From b12da23918ff94099e7fcedd7db79f3d2fa8fa6f Mon Sep 17 00:00:00 2001 From: Ogulcan Celik Date: Mon, 3 Aug 2026 05:20:15 +0300 Subject: [PATCH] feat(plugins): index marketplace manifests --- .gitignore | 1 + docs/next/CHANGELOG.md | 3 + .../src/content/docs/ja/marketplace.mdx | 33 +- .../website/src/content/docs/ja/plugins.mdx | 11 +- .../website/src/content/docs/marketplace.mdx | 35 +- .../next/website/src/content/docs/plugins.mdx | 13 +- .../src/content/docs/zh-cn/marketplace.mdx | 24 +- .../src/content/docs/zh-cn/plugins.mdx | 12 +- justfile | 2 +- workers/plugin-marketplace/bun.lock | 15 + workers/plugin-marketplace/package.json | 3 + workers/plugin-marketplace/src/index.test.ts | 737 ++++++++++---- workers/plugin-marketplace/src/index.ts | 945 +++++++++++++++--- workers/plugin-marketplace/wrangler.toml | 3 + 14 files changed, 1450 insertions(+), 387 deletions(-) create mode 100644 workers/plugin-marketplace/bun.lock diff --git a/.gitignore b/.gitignore index 3f6092c0..a93786db 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target +node_modules/ /vendor/libghostty-vt/zig-pkg/ __pycache__/ *.pyc diff --git a/docs/next/CHANGELOG.md b/docs/next/CHANGELOG.md index da9c9b71..7d1b9620 100644 --- a/docs/next/CHANGELOG.md +++ b/docs/next/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +### Added +- The plugin marketplace now discovers valid manifests at repository roots and subdirectories, groups multiple plugins under each repository, and publishes their versions and exact default-branch commits. + ## [0.8.0] - 2026-08-03 ### Added diff --git a/docs/next/website/src/content/docs/ja/marketplace.mdx b/docs/next/website/src/content/docs/ja/marketplace.mdx index 5f0a5c4c..5afb0675 100644 --- a/docs/next/website/src/content/docs/ja/marketplace.mdx +++ b/docs/next/website/src/content/docs/ja/marketplace.mdx @@ -9,10 +9,11 @@ Herdr プラグインマーケットプレイスは、コミュニティ製プ ## プラグインを探す -[マーケットプレイス](/plugins/)には、GitHub トピック `herdr-plugin` が付いた -すべての公開リポジトリが掲載されます。名前、オーナー、説明、言語で検索でき、 -人気順、最近の活動順、新着順で並べ替えられます。各掲載はソースリポジトリの -GitHub ページに直接リンクしています。 +[マーケットプレイス](/plugins/)には、GitHub トピック `herdr-plugin` が付いており、 +デフォルトブランチに必須メタデータを解析できる `herdr-plugin.toml` が 1 つ以上ある +公開リポジトリが掲載されます。リポジトリとプラグインのメタデータを検索でき、 +リポジトリカードを人気順、最近の活動順、新着順で並べ替えられます。各カードは +ソースリポジトリにリンクし、その中で検出されたすべてのプラグインを表示します。 掲載は自動かつ無審査です。掲載されているのはリポジトリが自らトピックを 付けたからであって、Herdr が検証したからではありません。インストールする前に @@ -35,19 +36,17 @@ herdr plugin install owner/repo[/subdir...] ## 自分のプラグインを掲載する -公開リポジトリに GitHub トピック `herdr-plugin` を追加してください。 -インデックスが使うシグナルはこのトピックだけなので、公開プラグインに -トピックを付けるだけで掲載されます。インデックスは 30 分ごとに自動更新される -ため、新しくトピックを付けたリポジトリはまもなく表示され、トピックを外した -リポジトリは次回の更新で消えます。 +公開リポジトリに GitHub トピック `herdr-plugin` を追加し、デフォルトブランチに +必須メタデータを解析できる `herdr-plugin.toml` を 1 つ以上置いてください。 +マニフェストはルートにもサブディレクトリにも配置できます。マーケットプレイスは +1 リポジトリにつき 1 枚のカードを使い、有効な各マニフェストを個別にインストール +できるプラグインとして表示します。インデックスは 30 分ごとに自動更新されます。 ## 掲載に表示される内容 -各カードには GitHub リポジトリのメタデータが表示されます: リポジトリ名と -オーナー、説明、スター数、主要言語、最終 push 時刻、そしてソースへのリンクです。 -インデックスは GitHub のリポジトリ検索からこれらを読み取るため、リポジトリの -説明とトピックを正確に保つことが、掲載を有用にする鍵になります。 - -インデックスはまだ `herdr-plugin.toml` を解析しないため、プラグインの `id`、 -宣言された `platforms`、`min_herdr_version` といったマニフェストのフィールドは -v1 では表示されません。フォークとアーカイブ済みリポジトリは一覧から除外されます。 +各カードには GitHub リポジトリの名前、オーナー、説明、スター数、主要言語、 +最終 push 時刻が表示されます。プラグイン行には各マニフェストの `name` と `version` +が表示され、正確なソースディレクトリにリンクします。インデックスにはパス、`id`、 +`name`、`version`、`platforms`、`min_herdr_version` と、デフォルトブランチの正確な +コミットが記録されます。フォーク、アーカイブ済みリポジトリ、有効なプラグイン +マニフェストがないリポジトリ、不正なメタデータは一覧から除外されます。 diff --git a/docs/next/website/src/content/docs/ja/plugins.mdx b/docs/next/website/src/content/docs/ja/plugins.mdx index db5342cd..aae2ad41 100644 --- a/docs/next/website/src/content/docs/ja/plugins.mdx +++ b/docs/next/website/src/content/docs/ja/plugins.mdx @@ -358,10 +358,13 @@ v1 には Herdr が管理するプラグインストレージ API はありま ## マーケットプレイス コミュニティ製プラグインは[マーケットプレイス](/plugins/)で探せます。これは -GitHub トピック `herdr-plugin` が付いた公開 GitHub リポジトリの自動インデックスです。 -プラグインは普通の GitHub リポジトリのままです: `herdr-plugin.toml` を含めて公開し、 +GitHub トピック `herdr-plugin` が付いており、必須メタデータを解析できる +`herdr-plugin.toml` が 1 つ以上ある公開 GitHub リポジトリの自動インデックスです。 +プラグインは普通の GitHub リポジトリのままです: 公開して `herdr plugin install owner/repo[/subdir]` を共有してください。 -プラグインを掲載するには、公開リポジトリに GitHub トピック `herdr-plugin` を -追加します。インデックスは 30 分ごとに更新されます。発見の仕組みは +プラグインを掲載するには、公開リポジトリに GitHub トピック `herdr-plugin` を追加し、 +デフォルトブランチのルートまたはサブディレクトリにマニフェストを置きます。 +1 つのリポジトリカードに複数のプラグインを掲載できます。インデックスは 30 分ごとに +更新されます。発見の仕組みは [マーケットプレイス](/ja/docs/marketplace/)を参照してください。 diff --git a/docs/next/website/src/content/docs/marketplace.mdx b/docs/next/website/src/content/docs/marketplace.mdx index 09a27b2d..88bbaf98 100644 --- a/docs/next/website/src/content/docs/marketplace.mdx +++ b/docs/next/website/src/content/docs/marketplace.mdx @@ -9,10 +9,12 @@ GitHub repositories, not a reviewed catalog. ## Browse plugins -The [marketplace](/plugins/) lists every public repository tagged with the -GitHub topic `herdr-plugin`. Search by name, owner, description, or language, -and sort by popularity, recent activity, or newest. Each listing links straight -to its source repository on GitHub. +The [marketplace](/plugins/) lists public repositories tagged with the GitHub +topic `herdr-plugin` when their default branch contains at least one +`herdr-plugin.toml` whose required metadata can be parsed. Search by repository +or plugin metadata, and sort repository cards by popularity, recent activity, +or newest. Each card links to its source repository and lists every discovered +plugin inside it. Discovery is automatic and unreviewed. A listing means a repository tagged itself, not that Herdr vetted it, so the @@ -34,20 +36,19 @@ its root, or in a subdirectory, and that command works. See ## Get your plugin listed -Add the GitHub topic `herdr-plugin` to a public repository. That topic is the -only signal the index uses, so tagging a public plugin is all it takes. The -index refreshes automatically every 30 minutes, so a newly tagged repository -shows up shortly after, and one that drops the topic disappears on the next -refresh. +Add the GitHub topic `herdr-plugin` to a public repository and put one or more +`herdr-plugin.toml` manifests with parseable required metadata on its default +branch. Manifests may be at the root or in subdirectories. The marketplace uses +one card per repository and lists each valid manifest as a separately +installable plugin. The index refreshes automatically every 30 minutes and +rescans repositories when their default-branch head changes. ## What a listing shows Each card shows GitHub repository metadata: the repository name and owner, its -description, star count, primary language, and the time it was last pushed, with -a link back to the source. The index reads this from GitHub's repository search, -so keeping your repository description and topics accurate is what makes your -listing useful. - -The index does not parse `herdr-plugin.toml` yet, so manifest fields like the -plugin `id`, declared `platforms`, and `min_herdr_version` are not shown in v1. -Forks and archived repositories are excluded from the list. +description, star count, primary language, and the time it was last pushed. Its +plugin rows show each manifest's `name` and `version` and link to the exact +source directory. The index records the manifest path, `id`, `name`, `version`, +`platforms`, and `min_herdr_version` together with the exact default-branch +commit. Forks, archived repositories, repositories without a valid plugin +manifest, and malformed manifest metadata are excluded. diff --git a/docs/next/website/src/content/docs/plugins.mdx b/docs/next/website/src/content/docs/plugins.mdx index 1d93bedc..418d5af6 100644 --- a/docs/next/website/src/content/docs/plugins.mdx +++ b/docs/next/website/src/content/docs/plugins.mdx @@ -363,10 +363,13 @@ state should own their files or database. ## Marketplace Community plugins are discoverable in the [marketplace](/plugins/), an automatic -index of public GitHub repositories tagged with the topic `herdr-plugin`. -Plugins stay ordinary GitHub repositories: publish one with `herdr-plugin.toml`, -then share `herdr plugin install owner/repo[/subdir]`. +index of public GitHub repositories tagged with `herdr-plugin` that contain one +or more `herdr-plugin.toml` files whose required metadata can be parsed. Plugins +stay ordinary GitHub repositories: publish one, then share +`herdr plugin install owner/repo[/subdir]`. -To get a plugin listed, add the GitHub topic `herdr-plugin` to its public -repository. The index refreshes every 30 minutes. See +To get plugins listed, add the GitHub topic `herdr-plugin` and place their +manifests at the root or in subdirectories of the repository's default branch. +One repository card can contain multiple separately installable plugins. The +index refreshes every 30 minutes. See [Marketplace](/docs/marketplace/) for how discovery works. diff --git a/docs/next/website/src/content/docs/zh-cn/marketplace.mdx b/docs/next/website/src/content/docs/zh-cn/marketplace.mdx index 350cc0d5..246048c7 100644 --- a/docs/next/website/src/content/docs/zh-cn/marketplace.mdx +++ b/docs/next/website/src/content/docs/zh-cn/marketplace.mdx @@ -9,9 +9,10 @@ Herdr 插件市场是一个可供发现的社区插件索引。 ## 浏览插件 -[插件市场](/plugins/)会列出所有打了 GitHub 主题标签 `herdr-plugin` 的公开仓库。 -你可以按名称、作者、描述或语言搜索,并按热度、最近活跃度或最新排序。 -每个条目都直接链接到它在 GitHub 上的源码仓库。 +[插件市场](/plugins/)会列出带有 GitHub 主题标签 `herdr-plugin`,并且默认分支中 +至少包含一个必需元数据可解析的 `herdr-plugin.toml` 的公开仓库。你可以搜索仓库和 +插件元数据,并按热度、最近活跃度或最新排序仓库卡片。每张卡片链接到源码仓库, +并列出其中发现的所有插件。 收录是自动且未经审核的。被列出只说明仓库给自己打了标签,并不代表 Herdr 审查过它,所以在安装任何插件之前,请先阅读[信任指南](/zh-cn/docs/plugins/#信任与安全)。 @@ -30,15 +31,14 @@ herdr plugin install owner/repo[/subdir...] ## 让你的插件被收录 -给公开仓库添加 GitHub 主题标签 `herdr-plugin`。索引只使用这一个信号, -所以给公开插件打上标签就够了。索引每 30 分钟自动刷新,新打标签的仓库很快 -就会出现,去掉标签的仓库会在下一次刷新时消失。 +给公开仓库添加 GitHub 主题标签 `herdr-plugin`,并在默认分支中放置一个或多个 +必需元数据可解析的 `herdr-plugin.toml`。清单可以位于根目录或子目录。插件市场 +每个仓库显示一张卡片,并把每份有效清单列为可单独安装的插件。索引每 30 分钟 +自动刷新。 ## 条目会展示什么 -每张卡片展示 GitHub 仓库的元数据: 仓库名和作者、描述、star 数、主要语言、 -最后 push 时间,以及指回源码的链接。索引从 GitHub 的仓库搜索读取这些信息, -所以保持仓库描述和主题标签的准确,是让条目有用的关键。 - -索引目前还不会解析 `herdr-plugin.toml`,所以插件 `id`、声明的 `platforms`、 -`min_herdr_version` 等清单字段在 v1 中不会展示。Fork 和已归档的仓库会被排除。 +每张卡片展示 GitHub 仓库的名称、作者、描述、star 数、主要语言和最后 push 时间。 +插件行展示每份清单的 `name` 和 `version`,并链接到精确的源码目录。索引还会记录 +清单路径、`id`、`name`、`version`、`platforms`、`min_herdr_version`,以及默认分支的 +精确提交。Fork、已归档仓库、没有有效插件清单的仓库以及无效清单元数据都会被排除。 diff --git a/docs/next/website/src/content/docs/zh-cn/plugins.mdx b/docs/next/website/src/content/docs/zh-cn/plugins.mdx index 306bda53..67c606f0 100644 --- a/docs/next/website/src/content/docs/zh-cn/plugins.mdx +++ b/docs/next/website/src/content/docs/zh-cn/plugins.mdx @@ -319,10 +319,12 @@ v1 没有 Herdr 管理的插件存储 API。需要持久状态的插件应自己 ## 插件市场 -社区插件可以在[插件市场](/plugins/)中发现,它是打了 `herdr-plugin` 主题 -标签的公开 GitHub 仓库的自动索引。插件仍然是普通的 GitHub 仓库: 发布一个 -带 `herdr-plugin.toml` 的仓库,然后分享 +社区插件可以在[插件市场](/plugins/)中发现,它是打了 `herdr-plugin` 主题标签且 +包含一个或多个必需元数据可解析的 `herdr-plugin.toml` 的公开 GitHub 仓库自动索引。 +插件仍然是普通的 GitHub 仓库: 发布后分享 `herdr plugin install owner/repo[/subdir]`。 -要让插件被收录,给它的公开仓库添加 GitHub 主题标签 `herdr-plugin`。索引 -每 30 分钟刷新一次。发现机制的工作方式见[插件市场](/zh-cn/docs/marketplace/)。 +要让插件被收录,给公开仓库添加 GitHub 主题标签 `herdr-plugin`,并在默认分支的 +根目录或子目录放置清单。一张仓库卡片可以包含多个插件。索引每 30 分钟刷新一次。 +发现机制的工作方式见 +[插件市场](/zh-cn/docs/marketplace/)。 diff --git a/justfile b/justfile index e5f9e1b2..a387f223 100644 --- a/justfile +++ b/justfile @@ -68,7 +68,7 @@ integration-assets-test: # Run plugin marketplace Worker tests plugin-marketplace-test: - cd workers/plugin-marketplace && bun test + cd workers/plugin-marketplace && bun install --frozen-lockfile && bun test # Build the vendored libghostty-vt source dist build-libghostty-vt: diff --git a/workers/plugin-marketplace/bun.lock b/workers/plugin-marketplace/bun.lock new file mode 100644 index 00000000..93e6ac52 --- /dev/null +++ b/workers/plugin-marketplace/bun.lock @@ -0,0 +1,15 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "herdr-plugin-marketplace-worker", + "dependencies": { + "smol-toml": "1.7.1", + }, + }, + }, + "packages": { + "smol-toml": ["smol-toml@1.7.1", "", {}, "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ=="], + } +} diff --git a/workers/plugin-marketplace/package.json b/workers/plugin-marketplace/package.json index d3ab29d1..a22f3179 100644 --- a/workers/plugin-marketplace/package.json +++ b/workers/plugin-marketplace/package.json @@ -6,5 +6,8 @@ "dev": "bunx wrangler dev", "deploy": "bunx wrangler deploy", "test": "bun test" + }, + "dependencies": { + "smol-toml": "1.7.1" } } diff --git a/workers/plugin-marketplace/src/index.test.ts b/workers/plugin-marketplace/src/index.test.ts index b84ed14d..9e3df19b 100644 --- a/workers/plugin-marketplace/src/index.test.ts +++ b/workers/plugin-marketplace/src/index.test.ts @@ -1,20 +1,44 @@ import { describe, expect, test } from "bun:test"; -import worker, { normalizeRepositories, refreshPlugins, type Env } from "./index"; +import worker, { + normalizeRepositories, + parseManifestSummary, + refreshPlugins, + type Env, +} from "./index"; + +const HEAD_COMMIT = "a".repeat(40); +const SECOND_COMMIT = "b".repeat(40); + +type TreeFixture = { + path: string; + content?: string; + mode?: string; + type?: string; + size?: number; +}; class MemoryR2 { objects = new Map(); - async put(key: string, value: string, options?: unknown): Promise { this.objects.set(key, { value, options }); } + + async get(key: string): Promise<{ text(): Promise } | null> { + const object = this.objects.get(key); + return object + ? { + async text() { + return object.value; + }, + } + : null; + } } class MemoryKV { constructor(private readonly keyNames: string[]) {} - async list(options?: { prefix?: string }): Promise<{ - keys: Array<{ name: string }>; - }> { + async list(options?: { prefix?: string }): Promise<{ keys: Array<{ name: string }> }> { return { keys: this.keyNames .filter((name) => !options?.prefix || name.startsWith(options.prefix)) @@ -29,8 +53,9 @@ function repo(overrides: Record = {}): Record full_name: "ogulcancelik/herdr-plugin-example", owner: { login: "ogulcancelik" }, name: "herdr-plugin-example", - description: "Example plugin", + description: "Example plugin repository", html_url: "https://github.com/ogulcancelik/herdr-plugin-example", + default_branch: "main", stargazers_count: 5, forks_count: 1, open_issues_count: 0, @@ -48,6 +73,17 @@ function repo(overrides: Record = {}): Record }; } +function manifest(overrides = ""): string { + return ` +id = "example.plugin" +name = "Example Plugin" +version = "0.2.0" +min_herdr_version = "0.7.0" +description = "Example manifest" +platforms = ["linux", "macos"] +${overrides}`; +} + function env(bucket = new MemoryR2(), blacklist?: MemoryKV): Env { return { PLUGIN_MARKETPLACE_BUCKET: bucket, @@ -56,62 +92,137 @@ function env(bucket = new MemoryR2(), blacklist?: MemoryKV): Env { }; } -describe("normalizeRepositories", () => { - test("normalizes repository fields into the public snapshot schema", () => { - const [plugin] = normalizeRepositories([repo()]); +function repositoryFetch(options: { + repositories: Record[]; + trees?: Record; + commits?: Record; + totalCount?: number; + incompleteResults?: boolean; + searchStatus?: number; + treeStatus?: Record; + truncatedTrees?: Set; + onRequest?: (kind: "search" | "head" | "tree" | "manifest", detail: string) => void; +}): typeof fetch { + return (async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = new URL(input.toString()); + if (url.pathname === "/search/repositories") { + options.onRequest?.("search", url.toString()); + if (options.searchStatus) return new Response("search failed", { status: options.searchStatus }); + return Response.json({ + total_count: options.totalCount ?? options.repositories.length, + incomplete_results: options.incompleteResults ?? false, + items: options.repositories, + }); + } - expect(plugin).toEqual({ + const treeMatch = url.pathname.match(/^\/repos\/([^/]+)\/([^/]+)\/git\/trees\/([^/]+)$/); + if (treeMatch) { + const fullName = `${decodeURIComponent(treeMatch[1])}/${decodeURIComponent(treeMatch[2])}`; + options.onRequest?.("tree", fullName); + const status = options.treeStatus?.[fullName]; + if (status) return new Response("tree failed", { status }); + const fixtures = options.trees?.[fullName] ?? [ + { path: "herdr-plugin.toml", content: manifest() }, + ]; + return Response.json({ + sha: "tree-sha", + truncated: options.truncatedTrees?.has(fullName) ?? false, + tree: fixtures.map((fixture) => ({ + path: fixture.path, + mode: fixture.mode ?? "100644", + type: fixture.type ?? "blob", + size: + fixture.size ?? + new TextEncoder().encode(fixture.content ?? "").byteLength, + })), + }); + } + + if (url.pathname === "/graphql") { + const request = JSON.parse(String(init?.body ?? "{}")); + const query = String(request.query ?? ""); + const data: Record = {}; + if (query.includes("PluginMarketplaceHeads")) { + options.onRequest?.("head", query); + for (const match of query.matchAll( + /repo(\d+): repository\(owner: "([^"]+)", name: "([^"]+)"\)/g, + )) { + const [, alias, owner, name] = match; + const fullName = `${owner}/${name}`; + const repository = options.repositories.find( + (candidate) => candidate.full_name === fullName, + ); + data[`repo${alias}`] = repository + ? { + defaultBranchRef: { + name: repository.default_branch ?? "main", + target: { oid: options.commits?.[fullName] ?? HEAD_COMMIT }, + }, + } + : null; + } + } else if (query.includes("PluginMarketplaceManifests")) { + options.onRequest?.("manifest", query); + for (const match of query.matchAll( + /item(\d+): repository\(owner: "([^"]+)", name: "([^"]+)"\) \{\s+manifest: object\(expression: "[a-f0-9]+:([^"]+)"\)/g, + )) { + const [, alias, owner, name, path] = match; + const fullName = `${owner}/${name}`; + const fixture = (options.trees?.[fullName] ?? [ + { path: "herdr-plugin.toml", content: manifest() }, + ]).find((entry) => entry.path === path); + data[`item${alias}`] = fixture?.content === undefined + ? { manifest: null } + : { manifest: { text: fixture.content } }; + } + } else { + throw new Error(`unexpected GraphQL query: ${query}`); + } + return Response.json({ data }); + } + + throw new Error(`unexpected request: ${url}`); + }) as typeof fetch; +} + +describe("normalizeRepositories", () => { + test("normalizes fields while preserving repository-card ordering", () => { + const plugins = normalizeRepositories([ + repo({ + id: 2, + full_name: "other/newer", + owner: { login: "other" }, + name: "newer", + html_url: "https://github.com/other/newer", + stargazers_count: 5, + pushed_at: "2026-06-04T00:00:00Z", + }), + repo(), + ]); + + expect(plugins.map((plugin) => plugin.fullName)).toEqual([ + "other/newer", + "ogulcancelik/herdr-plugin-example", + ]); + expect(plugins[1]).toMatchObject({ id: 1, - fullName: "ogulcancelik/herdr-plugin-example", owner: "ogulcancelik", name: "herdr-plugin-example", - description: "Example plugin", - url: "https://github.com/ogulcancelik/herdr-plugin-example", + defaultBranch: "main", stars: 5, - forks: 1, - openIssues: 0, - language: "TypeScript", - topics: ["herdr-plugin"], - createdAt: "2026-06-01T00:00:00Z", - updatedAt: "2026-06-02T00:00:00Z", - pushedAt: "2026-06-03T00:00:00Z", }); }); - test("sorts by stars, pushed date, and full name", () => { + test("deduplicates repositories by immutable GitHub id", () => { const plugins = normalizeRepositories([ - repo({ - id: 1, - full_name: "z/z", - owner: { login: "z" }, - name: "z", - html_url: "https://github.com/z/z", - stargazers_count: 3, - pushed_at: "2026-06-01T00:00:00Z", - }), - repo({ - id: 2, - full_name: "a/a", - owner: { login: "a" }, - name: "a", - html_url: "https://github.com/a/a", - stargazers_count: 3, - pushed_at: "2026-06-02T00:00:00Z", - }), - repo({ - id: 3, - full_name: "m/m", - owner: { login: "m" }, - name: "m", - html_url: "https://github.com/m/m", - stargazers_count: 10, - }), + repo(), + repo({ full_name: "duplicate/name", owner: { login: "duplicate" }, name: "name", html_url: "https://github.com/duplicate/name" }), ]); - - expect(plugins.map((plugin) => plugin.fullName)).toEqual(["m/m", "a/a", "z/z"]); + expect(plugins).toHaveLength(1); + expect(plugins[0].fullName).toBe("ogulcancelik/herdr-plugin-example"); }); - test("drops unsafe urls, archived repositories, forks, disabled repositories, and private repositories", () => { + test("drops unsafe, unavailable, and default-branch-less repositories", () => { const plugins = normalizeRepositories([ repo({ html_url: "https://example.com/ogulcancelik/herdr-plugin-example" }), repo({ archived: true }), @@ -119,62 +230,52 @@ describe("normalizeRepositories", () => { repo({ disabled: true }), repo({ private: true }), repo({ visibility: "private" }), + repo({ default_branch: undefined }), repo({ id: 5 }), ]); - expect(plugins.map((plugin) => plugin.id)).toEqual([5]); }); +}); - test("uses safe defaults for missing nullable fields", () => { - const [plugin] = normalizeRepositories([ - repo({ - id: undefined, - description: undefined, - stargazers_count: undefined, - forks_count: undefined, - open_issues_count: undefined, - language: undefined, - topics: undefined, - created_at: "not a date", - updated_at: undefined, - pushed_at: undefined, - }), - ]); +describe("parseManifestSummary", () => { + test("extracts metadata and accepts the UTF-8 BOM accepted by Herdr", () => { + expect(parseManifestSummary(`\uFEFF${manifest()}`)).toEqual({ + id: "example.plugin", + name: "Example Plugin", + version: "0.2.0", + minHerdrVersion: "0.7.0", + description: "Example manifest", + platforms: ["linux", "macos"], + }); + }); - expect(plugin.id).toBe(0); - expect(plugin.description).toBeNull(); - expect(plugin.stars).toBe(0); - expect(plugin.forks).toBe(0); - expect(plugin.openIssues).toBe(0); - expect(plugin.language).toBeNull(); - expect(plugin.topics).toEqual([]); - expect(plugin.createdAt).toBeNull(); - expect(plugin.updatedAt).toBeNull(); - expect(plugin.pushedAt).toBeNull(); + test("rejects malformed TOML and invalid required metadata", () => { + expect(parseManifestSummary("not = [valid")).toBeNull(); + expect(parseManifestSummary(manifest().replace("example.plugin", "bad/plugin"))).toBeNull(); + expect(parseManifestSummary(manifest().replace("0.7.0", "next"))).toBeNull(); + expect(parseManifestSummary(manifest().replace('["linux", "macos"]', "[]"))).toBeNull(); + expect( + parseManifestSummary(manifest().replace("Example Plugin", "n".repeat(121))), + ).toBeNull(); }); }); describe("refreshPlugins", () => { - test("fetches pages and writes a sorted snapshot to R2 with cache metadata", async () => { - const calls: string[] = []; - const fetch = async (input: RequestInfo | URL): Promise => { - const url = new URL(input.toString()); - calls.push(url.searchParams.get("page") ?? ""); - const page = url.searchParams.get("page"); - const item = - page === "1" - ? repo({ id: 1, full_name: "b/b", owner: { login: "b" }, name: "b", html_url: "https://github.com/b/b" }) - : repo({ - id: 2, - full_name: "a/a", - owner: { login: "a" }, - name: "a", - html_url: "https://github.com/a/a", - stargazers_count: 9, - }); - return Response.json({ total_count: 2, items: [item] }); - }; + test("publishes one backward-compatible repository card with multiple manifests", async () => { const bucket = new MemoryR2(); + const fullName = "ogulcancelik/herdr-plugin-example"; + const fetch = repositoryFetch({ + repositories: [repo()], + trees: { + [fullName]: [ + { path: "herdr-plugin.toml", content: manifest() }, + { + path: "plugins/second/herdr-plugin.toml", + content: manifest().replace("example.plugin", "example.second").replace("Example Plugin", "Second Plugin"), + }, + ], + }, + }); const result = await refreshPlugins(env(bucket), { fetch, @@ -183,142 +284,389 @@ describe("refreshPlugins", () => { }); expect(result.ok).toBe(true); - expect(calls).toEqual(["1", "2"]); - const object = bucket.objects.get("plugins/index.json"); - expect(object?.options).toEqual({ + const snapshotObject = bucket.objects.get("plugins/index.json"); + const snapshot = JSON.parse(snapshotObject?.value ?? ""); + expect(snapshotObject?.options).toEqual({ httpMetadata: { contentType: "application/json; charset=utf-8", cacheControl: "public, max-age=300, s-maxage=1800, stale-while-revalidate=3600", }, }); - const snapshot = JSON.parse(object?.value ?? ""); - expect(snapshot.generatedAt).toBe("2026-06-20T12:00:00.000Z"); - expect(snapshot.source).toMatchObject({ - provider: "github", - query: "topic:herdr-plugin is:public", - totalCount: 2, - collectedCount: 2, - truncated: false, + expect(snapshot).toMatchObject({ + schemaVersion: 1, + generatedAt: "2026-06-20T12:00:00.000Z", + pluginCount: 2, + repositoryCount: 1, + source: { + missingManifestCount: 0, + invalidManifestCount: 0, + }, }); - expect(snapshot.plugins.map((plugin: { fullName: string }) => plugin.fullName)).toEqual([ - "a/a", - "b/b", - ]); + expect(snapshot.plugins[0]).toMatchObject({ + id: 1, + fullName, + name: "herdr-plugin-example", + headCommit: HEAD_COMMIT, + manifests: [ + { path: "herdr-plugin.toml", id: "example.plugin" }, + { path: "plugins/second/herdr-plugin.toml", id: "example.second" }, + ], + }); + expect(snapshot.plugins[0]).not.toHaveProperty("defaultBranch"); + expect(bucket.objects.has("plugins/scan-cache.json")).toBe(true); + expect( + [...bucket.objects.keys()].some((key) => key.includes("history")), + ).toBe(false); }); - test("excludes repositories listed in the KV blacklist", async () => { - const fetch = async (): Promise => - Response.json({ - total_count: 2, - items: [ - repo({ - id: 1, - full_name: "example/not-a-plugin", - owner: { login: "example" }, - name: "not-a-plugin", - html_url: "https://github.com/example/not-a-plugin", - }), - repo({ - id: 2, - full_name: "ogulcancelik/herdr-plugin-example", - owner: { login: "ogulcancelik" }, - name: "herdr-plugin-example", - html_url: "https://github.com/ogulcancelik/herdr-plugin-example", - }), - ], - }); + test("reuses cached manifests without resolving or rescanning an unchanged repository", async () => { const bucket = new MemoryR2(); - - const result = await refreshPlugins(env(bucket, new MemoryKV(["repo:example/not-a-plugin"])), { - fetch, - logger: { error() {} }, + const requests: string[] = []; + const fetch = repositoryFetch({ + repositories: [repo()], + onRequest(kind) { + requests.push(kind); + }, }); - expect(result.ok).toBe(true); - const snapshot = JSON.parse(bucket.objects.get("plugins/index.json")?.value ?? ""); - expect(snapshot.plugins.map((plugin: { fullName: string }) => plugin.fullName)).toEqual([ - "ogulcancelik/herdr-plugin-example", - ]); + expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true); + requests.length = 0; + expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true); + + expect(requests).toEqual(["search"]); }); - test("writes an empty snapshot when every listable repository is blacklisted", async () => { - const fetch = async (): Promise => - Response.json({ - total_count: 1, - items: [ - repo({ - id: 1, - full_name: "example/not-a-plugin", - owner: { login: "example" }, - name: "not-a-plugin", - html_url: "https://github.com/example/not-a-plugin", - }), - ], - }); + test("rescans a repository when its pushed timestamp and head change", async () => { const bucket = new MemoryR2(); - await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}'); + const repository = repo(); + const fullName = String(repository.full_name); + const commits = { [fullName]: HEAD_COMMIT }; + const trees = { [fullName]: [{ path: "herdr-plugin.toml", content: manifest() }] }; + const fetch = repositoryFetch({ repositories: [repository], commits, trees }); - const result = await refreshPlugins(env(bucket, new MemoryKV(["repo:example/not-a-plugin"])), { - fetch, - logger: { error() {} }, - }); + expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true); + commits[fullName] = SECOND_COMMIT; + repository.pushed_at = "2026-06-04T00:00:00Z"; + trees[fullName][0].content = manifest().replace("0.2.0", "0.3.0"); + const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } }); expect(result.ok).toBe(true); - const snapshot = JSON.parse(bucket.objects.get("plugins/index.json")?.value ?? ""); - expect(snapshot.plugins).toEqual([]); + if (!result.ok) return; + expect(result.snapshot.plugins[0].headCommit).toBe(SECOND_COMMIT); + expect(result.snapshot.plugins[0].manifests[0].version).toBe("0.3.0"); }); - test("marks snapshots truncated at the GitHub search cap", async () => { - const fetch = async (): Promise => { - const items = Array.from({ length: 100 }, (_, index) => - repo({ - id: index, - full_name: `owner/repo-${index}`, - owner: { login: "owner" }, - name: `repo-${index}`, - html_url: `https://github.com/owner/repo-${index}`, - }), - ); - return Response.json({ total_count: 1200, items }); - }; - + test("counts manifests separately from repository cards and omits empty cards", async () => { + const repositories = [ + repo(), + repo({ + id: 2, + full_name: "example/empty", + owner: { login: "example" }, + name: "empty", + html_url: "https://github.com/example/empty", + }), + repo({ + id: 3, + full_name: "example/invalid", + owner: { login: "example" }, + name: "invalid", + html_url: "https://github.com/example/invalid", + }), + ]; const result = await refreshPlugins(env(), { - fetch, - now: new Date("2026-06-20T12:00:00.000Z"), + fetch: repositoryFetch({ + repositories, + trees: { + "ogulcancelik/herdr-plugin-example": [ + { path: "one/herdr-plugin.toml", content: manifest() }, + { path: "two/herdr-plugin.toml", content: manifest().replace("example.plugin", "example.two") }, + ], + "example/empty": [{ path: "README.md", content: "empty" }], + "example/invalid": [{ path: "herdr-plugin.toml", content: "id = [broken" }], + }, + }), logger: { error() {} }, }); expect(result.ok).toBe(true); if (!result.ok) return; - expect(result.snapshot.source.collectedCount).toBe(1000); - expect(result.snapshot.source.truncated).toBe(true); - expect(result.snapshot.source.warnings?.[0]).toContain("1200"); + expect(result.snapshot.pluginCount).toBe(2); + expect(result.snapshot.repositoryCount).toBe(1); + expect(result.snapshot.source.missingManifestCount).toBe(1); + expect(result.snapshot.source.invalidManifestCount).toBe(1); + }); + + test("ignores manifest symlinks and indexes their regular target", async () => { + const fullName = "ogulcancelik/herdr-plugin-example"; + const result = await refreshPlugins(env(), { + fetch: repositoryFetch({ + repositories: [repo()], + trees: { + [fullName]: [ + { path: "herdr-plugin.toml", content: "plugin/herdr-plugin.toml", mode: "120000" }, + { path: "plugin/herdr-plugin.toml", content: manifest() }, + ], + }, + }), + logger: { error() {} }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.snapshot.plugins[0].manifests.map((item) => item.path)).toEqual([ + "plugin/herdr-plugin.toml", + ]); + expect(result.snapshot.source.invalidManifestCount).toBe(0); + }); + + test("ignores test fixtures and deduplicates repeated plugin ids", async () => { + const fullName = "ogulcancelik/herdr-plugin-example"; + const result = await refreshPlugins(env(), { + fetch: repositoryFetch({ + repositories: [repo()], + trees: { + [fullName]: [ + { path: "herdr-plugin.toml", content: manifest() }, + { path: "platform/herdr-plugin.toml", content: manifest() }, + { + path: "tests/fixtures/helper/herdr-plugin.toml", + content: manifest().replace("example.plugin", "example.fixture"), + }, + ], + }, + }), + logger: { error() {} }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.snapshot.plugins[0].manifests.map((item) => item.path)).toEqual([ + "herdr-plugin.toml", + ]); + expect(result.snapshot.source.duplicateManifestCount).toBe(1); + }); + + test("skips a truncated tree without blocking the marketplace", async () => { + const fullName = "ogulcancelik/herdr-plugin-example"; + const result = await refreshPlugins(env(), { + fetch: repositoryFetch({ + repositories: [repo()], + truncatedTrees: new Set([fullName]), + }), + logger: { error() {} }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.snapshot.plugins).toEqual([]); + expect(result.snapshot.source.skippedRepositoryCount).toBe(1); + expect(result.snapshot.source.warnings?.[0]).toContain(fullName); + }); + + test("writes an empty snapshot when every repository is blacklisted", async () => { + const bucket = new MemoryR2(); + const result = await refreshPlugins( + env(bucket, new MemoryKV(["repo:ogulcancelik/herdr-plugin-example"])), + { fetch: repositoryFetch({ repositories: [repo()] }), logger: { error() {} } }, + ); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.snapshot.plugins).toEqual([]); + expect(result.snapshot.source.blacklistedCount).toBe(1); + }); + + test("pins tree and manifest reads to the resolved commit", async () => { + const requests: Array<[string, string]> = []; + const result = await refreshPlugins(env(), { + fetch: repositoryFetch({ + repositories: [repo()], + onRequest(kind, detail) { + requests.push([kind, detail]); + }, + }), + logger: { error() {} }, + }); + + expect(result.ok).toBe(true); + expect(requests.find(([kind]) => kind === "tree")?.[1]).toBe( + "ogulcancelik/herdr-plugin-example", + ); + const manifestQuery = requests.find(([kind]) => kind === "manifest")?.[1] ?? ""; + expect(manifestQuery).toContain(`${HEAD_COMMIT}:herdr-plugin.toml`); + expect(manifestQuery).not.toContain("main:herdr-plugin.toml"); + }); + + test("does not overwrite the public snapshot when scanning fails", async () => { + const bucket = new MemoryR2(); + await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}'); + const result = await refreshPlugins(env(bucket), { + fetch: repositoryFetch({ + repositories: [repo()], + treeStatus: { "ogulcancelik/herdr-plugin-example": 429 }, + }), + logger: { error() {} }, + }); + + expect(result.ok).toBe(false); + expect(bucket.objects.get("plugins/index.json")?.value).toBe( + '{"schemaVersion":1,"plugins":[{"id":1}]}', + ); + }); + + test("discards malformed cache state and rebuilds it", async () => { + const bucket = new MemoryR2(); + await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}'); + await bucket.put("plugins/scan-cache.json", "broken"); + const errors: string[] = []; + + const result = await refreshPlugins(env(bucket), { + fetch: repositoryFetch({ repositories: [repo()] }), + logger: { error(message) { errors.push(String(message)); } }, + }); + + expect(result.ok).toBe(true); + expect(errors[0]).toContain("discarding invalid plugin marketplace scan cache"); + expect(JSON.parse(bucket.objects.get("plugins/scan-cache.json")?.value ?? "").entries).toHaveLength(1); + }); + + test("rejects an implausible empty search when a healthy cache exists", async () => { + const bucket = new MemoryR2(); + const normalFetch = repositoryFetch({ repositories: [repo()] }); + expect((await refreshPlugins(env(bucket), { fetch: normalFetch, logger: { error() {} } })).ok).toBe(true); + const previous = bucket.objects.get("plugins/index.json")?.value; + + const result = await refreshPlugins(env(bucket), { + fetch: repositoryFetch({ repositories: [] }), + logger: { error() {} }, + }); + + expect(result.ok).toBe(false); + expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous); + }); + + test("publishes an empty snapshot when the initial complete search is empty", async () => { + const bucket = new MemoryR2(); + const result = await refreshPlugins(env(bucket), { + fetch: repositoryFetch({ repositories: [] }), + logger: { error() {} }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.snapshot.plugins).toEqual([]); + expect(result.snapshot.pluginCount).toBe(0); + }); + + test("preserves the snapshot when head resolution omits a cached repository", async () => { + const bucket = new MemoryR2(); + const repository = repo(); + const baseFetch = repositoryFetch({ repositories: [repository] }); + expect((await refreshPlugins(env(bucket), { fetch: baseFetch, logger: { error() {} } })).ok).toBe(true); + const previous = bucket.objects.get("plugins/index.json")?.value; + + repository.pushed_at = "2026-06-04T00:00:00Z"; + const missingHeadFetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const request = String(init?.body ?? ""); + if (new URL(input.toString()).pathname === "/graphql" && request.includes("PluginMarketplaceHeads")) { + return Response.json({ + data: { repo0: null }, + errors: [{ type: "NOT_FOUND", path: ["repo0"], message: "not found" }], + }); + } + return baseFetch(input, init); + }; + const result = await refreshPlugins(env(bucket), { + fetch: missingHeadFetch as typeof fetch, + logger: { error() {} }, + }); + + expect(result.ok).toBe(false); + expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous); + }); + + test("does not cache a transiently missing manifest response", async () => { + const bucket = new MemoryR2(); + const repository = repo(); + const fullName = String(repository.full_name); + const commits = { [fullName]: HEAD_COMMIT }; + const baseFetch = repositoryFetch({ repositories: [repository], commits }); + expect((await refreshPlugins(env(bucket), { fetch: baseFetch, logger: { error() {} } })).ok).toBe(true); + const previous = bucket.objects.get("plugins/index.json")?.value; + + repository.pushed_at = "2026-06-04T00:00:00Z"; + commits[fullName] = SECOND_COMMIT; + const missingFetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const request = String(init?.body ?? ""); + if (new URL(input.toString()).pathname === "/graphql" && request.includes("PluginMarketplaceManifests")) { + return Response.json({ + data: { item0: { manifest: null } }, + errors: [{ type: "NOT_FOUND", path: ["item0"], message: "not found" }], + }); + } + return baseFetch(input, init); + }; + const result = await refreshPlugins(env(bucket), { + fetch: missingFetch as typeof fetch, + logger: { error() {} }, + }); + + expect(result.ok).toBe(false); + expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous); + }); + + test("retries budget-skipped repositories when capacity becomes available", async () => { + const repositories = Array.from({ length: 51 }, (_, index) => + repo({ + id: index + 1, + full_name: `owner/repo-${index}`, + owner: { login: "owner" }, + name: `repo-${index}`, + html_url: `https://github.com/owner/repo-${index}`, + }), + ); + const trees = Object.fromEntries( + repositories.map((repository) => [ + repository.full_name, + Array.from({ length: 100 }, (_, index) => ({ + path: `plugins/${index}/herdr-plugin.toml`, + content: manifest(), + })), + ]), + ); + + const bucket = new MemoryR2(); + const fetch = repositoryFetch({ repositories, trees }); + const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.snapshot.source.skippedRepositoryCount).toBe(1); + expect(result.snapshot.source.warnings?.[0]).toContain("scan budget was exhausted"); + + repositories.shift(); + const recovered = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } }); + expect(recovered.ok).toBe(true); + if (!recovered.ok) return; + expect(recovered.snapshot.source.skippedRepositoryCount).toBe(0); + expect(recovered.snapshot.repositoryCount).toBe(50); }); for (const { name, fetch } of [ { name: "GitHub failure", - fetch: async (): Promise => new Response("rate limited", { status: 429 }), - }, - { - name: "no listable repositories", - fetch: async (): Promise => Response.json({ total_count: 0, items: [] }), + fetch: repositoryFetch({ repositories: [], searchStatus: 429 }), }, { name: "incomplete search results", - fetch: async (): Promise => - Response.json({ total_count: 1, incomplete_results: true, items: [repo()] }), + fetch: repositoryFetch({ repositories: [repo()], incompleteResults: true }), }, ]) { - test(`does not overwrite the R2 snapshot on ${name}`, async () => { + test(`preserves the current snapshot on ${name}`, async () => { const bucket = new MemoryR2(); await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}'); - - const result = await refreshPlugins(env(bucket), { - fetch, - logger: { error() {} }, - }); - + const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } }); expect(result.ok).toBe(false); expect(bucket.objects.get("plugins/index.json")?.value).toBe( '{"schemaVersion":1,"plugins":[{"id":1}]}', @@ -330,7 +678,6 @@ describe("refreshPlugins", () => { describe("fetch handler", () => { test("does not expose a public Worker API", async () => { const response = await worker.fetch(new Request("https://herdr.dev/api/plugins"), env()); - expect(response.status).toBe(404); expect(response.headers.get("Cache-Control")).toBe("no-store"); }); diff --git a/workers/plugin-marketplace/src/index.ts b/workers/plugin-marketplace/src/index.ts index e4523340..a504e1af 100644 --- a/workers/plugin-marketplace/src/index.ts +++ b/workers/plugin-marketplace/src/index.ts @@ -1,12 +1,32 @@ +import { parse } from "smol-toml"; + const SNAPSHOT_KEY = "plugins/index.json"; +const SCAN_CACHE_KEY = "plugins/scan-cache.json"; const SNAPSHOT_CACHE_CONTROL = "public, max-age=300, s-maxage=1800, stale-while-revalidate=3600"; const GITHUB_QUERY = "topic:herdr-plugin is:public"; const GITHUB_API_VERSION = "2022-11-28"; -const GITHUB_SEARCH_URL = "https://api.github.com/search/repositories"; +const GITHUB_API_URL = "https://api.github.com"; +const GITHUB_SEARCH_URL = `${GITHUB_API_URL}/search/repositories`; +const GITHUB_GRAPHQL_URL = `${GITHUB_API_URL}/graphql`; const BLACKLIST_REPO_KEY_PREFIX = "repo:"; +const PLUGIN_MANIFEST_FILE = "herdr-plugin.toml"; const PER_PAGE = 100; const MAX_REPOS = 1000; +const GRAPHQL_BATCH_SIZE = 50; +const NETWORK_CONCURRENCY = 5; +const MANIFEST_MAX_BYTES = 32 * 1024; +const MAX_TREE_ENTRIES = 20_000; +const MAX_MANIFESTS_PER_REPOSITORY = 100; +const MAX_TOTAL_MANIFESTS = 5000; +const MAX_TOTAL_MANIFEST_BYTES = 16 * 1024 * 1024; +const PLUGIN_NAME_MAX_CHARS = 120; +const PLUGIN_VERSION_MAX_CHARS = 64; +const PLUGIN_DESCRIPTION_MAX_CHARS = 500; const REQUEST_TIMEOUT_MS = 10_000; +const SCAN_CACHE_SCHEMA_VERSION = 1; + +const PLUGIN_PLATFORMS = new Set(["linux", "macos", "windows"]); +const REGULAR_BLOB_MODES = new Set(["100644", "100755"]); type R2Bucket = { put( @@ -19,6 +39,7 @@ type R2Bucket = { }; }, ): Promise; + get(key: string): Promise<{ text(): Promise } | null>; }; type KVNamespace = { @@ -50,7 +71,7 @@ type RefreshOptions = { type GitHubRepository = Record; -export type PluginListing = { +type RepositoryListing = { id: number; fullName: string; owner: string; @@ -65,22 +86,94 @@ export type PluginListing = { createdAt: string | null; updatedAt: string | null; pushedAt: string | null; + defaultBranch: string; +}; + +export type PluginManifestListing = { + path: string; + id: string; + name: string; + version: string; + minHerdrVersion: string; + description: string | null; + platforms: string[] | null; +}; + +export type PluginListing = Omit & { + headCommit: string; + manifests: PluginManifestListing[]; }; export type PluginSnapshot = { schemaVersion: 1; generatedAt: string; + pluginCount: number; + repositoryCount: number; source: { provider: "github"; query: string; totalCount: number; collectedCount: number; + blacklistedCount: number; + missingManifestCount: number; + invalidManifestCount: number; + duplicateManifestCount: number; + skippedRepositoryCount: number; + unavailableRepositoryCount: number; truncated: boolean; warnings?: string[]; }; plugins: PluginListing[]; }; +type ScanCacheEntry = { + repositoryId: number; + fullName: string; + defaultBranch: string; + pushedAt: string | null; + headCommit: string; + manifestFileCount: number; + invalidManifestCount: number; + duplicateManifestCount: number; + warning: string | null; + retryable: boolean; + manifests: PluginManifestListing[]; +}; + +type ScanCache = { + schemaVersion: 1; + entries: ScanCacheEntry[]; +}; + +type ResolvedRepository = { + repository: RepositoryListing; + headCommit: string; + cached: ScanCacheEntry | null; +}; + +type TreeManifestCandidate = { + repository: RepositoryListing; + headCommit: string; + path: string; + size: number; +}; + +type PendingRepositoryScan = { + resolved: ResolvedRepository; + manifestFileCount: number; + invalidManifestCount: number; + warning: string | null; + retryable: boolean; + candidates: TreeManifestCandidate[]; +}; + +type GitHubTreeEntry = { + path: string; + mode: string; + type: string; + size: number | null; +}; + export type RefreshResult = | { ok: true; snapshot: PluginSnapshot } | { ok: false; error: string }; @@ -107,39 +200,112 @@ export async function refreshPlugins( } const fetchFn = options.fetch ?? fetch; - const result = await fetchGitHubRepositories(fetchFn, token); - const normalizedPlugins = normalizeRepositories(result.repositories); - if (normalizedPlugins.length === 0) { - throw new Error("GitHub returned no listable plugin repositories"); - } - + const search = await fetchGitHubRepositories(fetchFn, token); + const repositories = normalizeRepositories(search.repositories); const blockedRepositories = await readBlacklistedRepositories(env); - const plugins = - blockedRepositories.size === 0 - ? normalizedPlugins - : normalizedPlugins.filter( - (plugin) => !blockedRepositories.has(plugin.fullName.toLowerCase()), - ); + const candidates = repositories.filter( + (repository) => !blockedRepositories.has(repository.fullName.toLowerCase()), + ); + const blacklistedCount = repositories.length - candidates.length; + const cache = await readScanCache(env.PLUGIN_MARKETPLACE_BUCKET, logger); + if (cache.entries.length > 0 && repositories.length * 2 < cache.entries.length) { + throw new Error( + `GitHub repository inventory collapsed from ${cache.entries.length} to ${repositories.length}; remove ${SCAN_CACHE_KEY} to accept an intentional reset`, + ); + } + const cachedById = new Map(cache.entries.map((entry) => [entry.repositoryId, entry])); + const { resolved, unavailableRepositoryCount } = await resolveRepositoryHeads( + fetchFn, + token, + candidates, + cachedById, + ); + const resolvedIds = new Set(resolved.map(({ repository }) => repository.id)); + const missingCachedRepository = candidates.find( + (repository) => cachedById.has(repository.id) && !resolvedIds.has(repository.id), + ); + if ( + missingCachedRepository || + (candidates.length > 0 && resolved.length * 2 < candidates.length) + ) { + throw new Error( + `GitHub head resolution omitted marketplace repositories${missingCachedRepository ? ` including ${missingCachedRepository.fullName}` : ""}`, + ); + } + const changed = resolved.filter( + ({ headCommit, cached }) => !cached || cached.headCommit !== headCommit || cached.retryable, + ); + const scannedById = await scanChangedRepositories(fetchFn, token, changed); + + const nextEntries = resolved.map(({ repository, headCommit, cached }) => { + const scanned = scannedById.get(repository.id); + if (scanned) return scanned; + if (!cached || cached.headCommit !== headCommit) { + throw new Error(`missing marketplace scan result for ${repository.fullName}`); + } + return { + ...cached, + fullName: repository.fullName, + defaultBranch: repository.defaultBranch, + pushedAt: repository.pushedAt, + }; + }); + const entriesById = new Map(nextEntries.map((entry) => [entry.repositoryId, entry])); + const plugins = resolved + .map(({ repository }) => { + const entry = entriesById.get(repository.id); + if (!entry || entry.manifests.length === 0) return null; + const { defaultBranch: _defaultBranch, ...card } = repository; + return { ...card, headCommit: entry.headCommit, manifests: entry.manifests }; + }) + .filter((plugin): plugin is PluginListing => plugin !== null); + + const pluginCount = plugins.reduce((total, plugin) => total + plugin.manifests.length, 0); + const warnings = nextEntries.flatMap((entry) => (entry.warning ? [entry.warning] : [])); + if (search.truncated) { + warnings.unshift( + `GitHub returned ${search.totalCount} results; only the first ${search.repositories.length} were collected.`, + ); + } + const generatedAt = (options.now ?? new Date()).toISOString(); const snapshot: PluginSnapshot = { schemaVersion: 1, - generatedAt: (options.now ?? new Date()).toISOString(), + generatedAt, + pluginCount, + repositoryCount: plugins.length, source: { provider: "github", query: GITHUB_QUERY, - totalCount: result.totalCount, - collectedCount: result.repositories.length, - truncated: result.truncated, + totalCount: search.totalCount, + collectedCount: search.repositories.length, + blacklistedCount, + missingManifestCount: nextEntries.filter( + (entry) => entry.manifestFileCount === 0 && !entry.warning, + ).length, + invalidManifestCount: nextEntries.reduce( + (total, entry) => total + entry.invalidManifestCount, + 0, + ), + duplicateManifestCount: nextEntries.reduce( + (total, entry) => total + entry.duplicateManifestCount, + 0, + ), + skippedRepositoryCount: nextEntries.filter((entry) => entry.warning).length, + unavailableRepositoryCount, + truncated: search.truncated, + ...(warnings.length > 0 ? { warnings } : {}), }, plugins, }; - if (result.truncated) { - snapshot.source.warnings = [ - `GitHub returned ${result.totalCount} results; only the first ${result.repositories.length} were collected.`, - ]; - } - + const nextCache: ScanCache = { schemaVersion: SCAN_CACHE_SCHEMA_VERSION, entries: nextEntries }; + await env.PLUGIN_MARKETPLACE_BUCKET.put(SCAN_CACHE_KEY, JSON.stringify(nextCache), { + httpMetadata: { + contentType: "application/json; charset=utf-8", + cacheControl: "no-store", + }, + }); await env.PLUGIN_MARKETPLACE_BUCKET.put(SNAPSHOT_KEY, JSON.stringify(snapshot), { httpMetadata: { contentType: "application/json; charset=utf-8", @@ -170,24 +336,10 @@ async function fetchGitHubRepositories( url.searchParams.set("sort", "stars"); url.searchParams.set("order", "desc"); - const response = await fetchWithTimeout( - fetchFn, - url, - { - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "User-Agent": "herdr-plugin-marketplace", - "X-GitHub-Api-Version": GITHUB_API_VERSION, - }, - }, - timeoutMs, - ); - + const response = await fetchWithTimeout(fetchFn, url, githubRequestInit(token), timeoutMs); if (!response.ok) { throw new Error(`GitHub search failed with status ${response.status}`); } - const body = await response.json(); if (!isObject(body) || typeof body.total_count !== "number" || !Array.isArray(body.items)) { throw new Error("GitHub search returned malformed JSON"); @@ -198,10 +350,7 @@ async function fetchGitHubRepositories( totalCount = body.total_count; repositories.push(...body.items.slice(0, MAX_REPOS - repositories.length)); - - if (repositories.length >= totalCount || body.items.length === 0) { - break; - } + if (repositories.length >= totalCount || body.items.length === 0) break; } return { @@ -211,6 +360,579 @@ async function fetchGitHubRepositories( }; } +async function resolveRepositoryHeads( + fetchFn: FetchLike, + token: string, + repositories: RepositoryListing[], + cachedById: Map, +): Promise<{ resolved: ResolvedRepository[]; unavailableRepositoryCount: number }> { + const resolvedById = new Map(); + const needResolution: RepositoryListing[] = []; + + for (const repository of repositories) { + const cached = cachedById.get(repository.id) ?? null; + if ( + cached && + cached.fullName.toLowerCase() === repository.fullName.toLowerCase() && + cached.defaultBranch === repository.defaultBranch && + !cached.retryable && + cached.pushedAt !== null && + cached.pushedAt === repository.pushedAt + ) { + resolvedById.set(repository.id, { repository, headCommit: cached.headCommit, cached }); + } else { + needResolution.push(repository); + } + } + + const batches = chunk(needResolution, GRAPHQL_BATCH_SIZE); + const results = await mapConcurrent(batches, NETWORK_CONCURRENCY, async (batch) => { + const data = await fetchGraphqlData(fetchFn, token, repositoryHeadQuery(batch)); + return batch.flatMap((repository, index) => { + const raw = data[`repo${index}`]; + if (!isObject(raw)) return []; + const branch = isObject(raw.defaultBranchRef) ? readString(raw.defaultBranchRef.name) : null; + const target = isObject(raw.defaultBranchRef?.target) ? raw.defaultBranchRef.target : null; + const headCommit = isObject(target) ? readString(target.oid) : null; + if ( + branch !== repository.defaultBranch || + !headCommit || + !/^[0-9a-f]{40,64}$/i.test(headCommit) + ) { + return []; + } + return [{ repository, headCommit, cached: cachedById.get(repository.id) ?? null }]; + }); + }); + for (const resolved of results.flat()) { + resolvedById.set(resolved.repository.id, resolved); + } + + return { + resolved: repositories.flatMap((repository) => { + const value = resolvedById.get(repository.id); + return value ? [value] : []; + }), + unavailableRepositoryCount: repositories.length - resolvedById.size, + }; +} + +async function scanChangedRepositories( + fetchFn: FetchLike, + token: string, + repositories: ResolvedRepository[], +): Promise> { + const pending = await mapConcurrent(repositories, NETWORK_CONCURRENCY, (resolved) => + fetchRepositoryTree(fetchFn, token, resolved), + ); + const candidates: TreeManifestCandidate[] = []; + let candidateBytes = 0; + for (const scan of pending) { + const scanBytes = scan.candidates.reduce((total, candidate) => total + candidate.size, 0); + if ( + candidates.length + scan.candidates.length > MAX_TOTAL_MANIFESTS || + candidateBytes + scanBytes > MAX_TOTAL_MANIFEST_BYTES + ) { + scan.warning = `${scan.resolved.repository.fullName} was skipped because the marketplace scan budget was exhausted.`; + scan.retryable = true; + scan.candidates = []; + continue; + } + candidates.push(...scan.candidates); + candidateBytes += scanBytes; + } + const contents = await fetchManifestContents(fetchFn, token, candidates); + const entries = new Map(); + + for (const scan of pending) { + const manifests: PluginManifestListing[] = []; + let invalidManifestCount = scan.invalidManifestCount; + for (const candidate of scan.candidates) { + const content = contents.get(manifestCandidateKey(candidate)); + const metadata = typeof content === "string" ? parseManifestSummary(content) : null; + if (!metadata) { + invalidManifestCount += 1; + continue; + } + manifests.push({ path: candidate.path, ...metadata }); + } + const { manifests: uniqueManifests, duplicateCount } = deduplicateManifests(manifests); + const { repository, headCommit } = scan.resolved; + entries.set(repository.id, { + repositoryId: repository.id, + fullName: repository.fullName, + defaultBranch: repository.defaultBranch, + pushedAt: repository.pushedAt, + headCommit, + manifestFileCount: scan.manifestFileCount, + invalidManifestCount, + duplicateManifestCount: duplicateCount, + warning: scan.warning, + retryable: scan.retryable, + manifests: uniqueManifests, + }); + } + return entries; +} + +async function fetchRepositoryTree( + fetchFn: FetchLike, + token: string, + resolved: ResolvedRepository, +): Promise { + const { repository, headCommit } = resolved; + const url = new URL( + `${GITHUB_API_URL}/repos/${encodeURIComponent(repository.owner)}/${encodeURIComponent(repository.name)}/git/trees/${headCommit}`, + ); + url.searchParams.set("recursive", "1"); + const response = await fetchWithTimeout(fetchFn, url, githubRequestInit(token), REQUEST_TIMEOUT_MS); + if (!response.ok) { + throw new Error(`GitHub tree fetch failed for ${repository.fullName} with status ${response.status}`); + } + const body = await response.json(); + if (!isObject(body) || !Array.isArray(body.tree)) { + throw new Error(`GitHub tree fetch returned malformed JSON for ${repository.fullName}`); + } + if (body.truncated === true || body.tree.length > MAX_TREE_ENTRIES) { + return skippedRepositoryScan(resolved, `${repository.fullName} was skipped because its Git tree is too large.`); + } + + const manifestEntries = body.tree + .map(normalizeTreeEntry) + .filter((entry): entry is GitHubTreeEntry => entry !== null) + .filter( + (entry) => + isDiscoverableManifestPath(entry.path) && + entry.type === "blob" && + REGULAR_BLOB_MODES.has(entry.mode), + ); + if (manifestEntries.length > MAX_MANIFESTS_PER_REPOSITORY) { + return skippedRepositoryScan( + resolved, + `${repository.fullName} was skipped because it contains more than ${MAX_MANIFESTS_PER_REPOSITORY} plugin manifests.`, + ); + } + + let invalidManifestCount = 0; + const candidates: TreeManifestCandidate[] = []; + for (const entry of manifestEntries) { + if ( + entry.size === null || + entry.size < 0 || + entry.size > MANIFEST_MAX_BYTES + ) { + invalidManifestCount += 1; + continue; + } + candidates.push({ repository, headCommit, path: entry.path, size: entry.size }); + } + + return { + resolved, + manifestFileCount: manifestEntries.length, + invalidManifestCount, + warning: null, + retryable: false, + candidates, + }; +} + +function skippedRepositoryScan( + resolved: ResolvedRepository, + warning: string, +): PendingRepositoryScan { + return { + resolved, + manifestFileCount: 0, + invalidManifestCount: 0, + warning, + retryable: false, + candidates: [], + }; +} + +async function fetchManifestContents( + fetchFn: FetchLike, + token: string, + candidates: TreeManifestCandidate[], +): Promise> { + const batches = chunk(candidates, GRAPHQL_BATCH_SIZE); + const results = await mapConcurrent(batches, NETWORK_CONCURRENCY, async (batch) => { + const data = await fetchGraphqlData(fetchFn, token, manifestContentQuery(batch)); + return batch.flatMap((candidate, index) => { + const raw = data[`item${index}`]; + const manifest = isObject(raw) && isObject(raw.manifest) ? raw.manifest : null; + if (!manifest) { + throw new Error( + `GitHub manifest fetch omitted ${candidate.repository.fullName}:${candidate.path}`, + ); + } + if (manifest.isBinary === true) { + return [[manifestCandidateKey(candidate), null] as const]; + } + if (typeof manifest.text !== "string") { + throw new Error( + `GitHub manifest fetch returned malformed data for ${candidate.repository.fullName}:${candidate.path}`, + ); + } + return [[manifestCandidateKey(candidate), manifest.text] as const]; + }); + }); + return new Map(results.flat()); +} + +async function fetchGraphqlData( + fetchFn: FetchLike, + token: string, + query: string, +): Promise> { + const response = await fetchWithTimeout( + fetchFn, + new URL(GITHUB_GRAPHQL_URL), + { + ...githubRequestInit(token), + method: "POST", + body: JSON.stringify({ query }), + }, + REQUEST_TIMEOUT_MS, + ); + if (!response.ok) { + throw new Error(`GitHub marketplace query failed with status ${response.status}`); + } + const body = await response.json(); + if (!isObject(body) || !isObject(body.data)) { + throw new Error("GitHub marketplace query returned malformed JSON"); + } + if ( + Array.isArray(body.errors) && + body.errors.some((error) => !isMissingAliasGraphqlError(error)) + ) { + throw new Error("GitHub marketplace query returned GraphQL errors"); + } + return body.data; +} + +function isMissingAliasGraphqlError(error: unknown): boolean { + if (!isObject(error) || error.type !== "NOT_FOUND" || !Array.isArray(error.path)) { + return false; + } + const alias = error.path[0]; + return typeof alias === "string" && /^(repo|item)\d+$/.test(alias); +} + +function repositoryHeadQuery(repositories: RepositoryListing[]): string { + const selections = repositories.map((repository, index) => { + const owner = JSON.stringify(repository.owner); + const name = JSON.stringify(repository.name); + return `repo${index}: repository(owner: ${owner}, name: ${name}) { + defaultBranchRef { name target { ... on Commit { oid } } } + }`; + }); + return `query PluginMarketplaceHeads {\n${selections.join("\n")}\n}`; +} + +function manifestContentQuery(candidates: TreeManifestCandidate[]): string { + const selections = candidates.map((candidate, index) => { + const owner = JSON.stringify(candidate.repository.owner); + const name = JSON.stringify(candidate.repository.name); + const expression = JSON.stringify(`${candidate.headCommit}:${candidate.path}`); + return `item${index}: repository(owner: ${owner}, name: ${name}) { + manifest: object(expression: ${expression}) { ... on Blob { text isBinary } } + }`; + }); + return `query PluginMarketplaceManifests {\n${selections.join("\n")}\n}`; +} + +export function parseManifestSummary(manifestText: string): Omit | null { + try { + const manifest = parse(manifestText.replace(/^\uFEFF/, "")); + if (!isObject(manifest)) return null; + const id = readTrimmedString(manifest.id); + const name = readTrimmedString(manifest.name); + const version = readTrimmedString(manifest.version); + const minHerdrVersion = readTrimmedString(manifest.min_herdr_version); + if ( + !id || + id.length > 120 || + !/^[A-Za-z0-9:._-]+$/.test(id) || + !name || + name.length > PLUGIN_NAME_MAX_CHARS || + !version || + version.length > PLUGIN_VERSION_MAX_CHARS || + !minHerdrVersion || + minHerdrVersion.length > PLUGIN_VERSION_MAX_CHARS || + !isHerdrVersion(minHerdrVersion) + ) { + return null; + } + + let description: string | null = null; + if (manifest.description !== undefined) { + if (typeof manifest.description !== "string") return null; + description = manifest.description.trim() || null; + if (description && description.length > PLUGIN_DESCRIPTION_MAX_CHARS) return null; + } + + let platforms: string[] | null = null; + if (manifest.platforms !== undefined) { + if ( + !Array.isArray(manifest.platforms) || + manifest.platforms.length === 0 || + manifest.platforms.length > PLUGIN_PLATFORMS.size || + !manifest.platforms.every( + (platform) => typeof platform === "string" && PLUGIN_PLATFORMS.has(platform), + ) + ) { + return null; + } + platforms = [...manifest.platforms]; + if (new Set(platforms).size !== platforms.length) return null; + } + + return { id, name, version, minHerdrVersion, description, platforms }; + } catch { + return null; + } +} + +async function readScanCache( + bucket: R2Bucket, + logger: Pick, +): Promise { + const empty = (): ScanCache => ({ schemaVersion: SCAN_CACHE_SCHEMA_VERSION, entries: [] }); + const object = await bucket.get(SCAN_CACHE_KEY); + if (!object) return empty(); + try { + const value: unknown = JSON.parse(await object.text()); + if ( + !isObject(value) || + value.schemaVersion !== SCAN_CACHE_SCHEMA_VERSION || + !Array.isArray(value.entries) + ) { + throw new Error("unsupported cache shape"); + } + const entries = value.entries.map(readScanCacheEntry); + if (entries.some((entry) => entry === null)) { + throw new Error("invalid cache entry"); + } + return { schemaVersion: SCAN_CACHE_SCHEMA_VERSION, entries: entries as ScanCacheEntry[] }; + } catch (error) { + const message = error instanceof Error ? error.message : "unknown cache error"; + logger.error(`discarding invalid plugin marketplace scan cache: ${message}`); + return empty(); + } +} + +function readScanCacheEntry(value: unknown): ScanCacheEntry | null { + if (!isObject(value) || !Array.isArray(value.manifests)) return null; + const repositoryId = readInteger(value.repositoryId); + const fullName = readString(value.fullName); + const defaultBranch = readString(value.defaultBranch); + const pushedAt = value.pushedAt === null ? null : readIsoString(value.pushedAt); + const headCommit = readString(value.headCommit); + const manifestFileCount = readNonNegativeIntegerOrNull(value.manifestFileCount); + const invalidManifestCount = readNonNegativeIntegerOrNull(value.invalidManifestCount); + const duplicateManifestCount = readNonNegativeIntegerOrNull(value.duplicateManifestCount); + const warning = value.warning === null ? null : readString(value.warning); + const retryable = typeof value.retryable === "boolean" ? value.retryable : null; + const manifests = value.manifests.map(readCachedManifest); + if ( + repositoryId === null || + !fullName || + !defaultBranch || + (value.pushedAt !== null && pushedAt === null) || + !headCommit || + !/^[0-9a-f]{40,64}$/i.test(headCommit) || + manifestFileCount === null || + invalidManifestCount === null || + duplicateManifestCount === null || + retryable === null || + (value.warning !== null && !warning) || + manifests.some((manifest) => manifest === null) + ) { + return null; + } + return { + repositoryId, + fullName, + defaultBranch, + pushedAt, + headCommit, + manifestFileCount, + invalidManifestCount, + duplicateManifestCount, + warning, + retryable, + manifests: manifests as PluginManifestListing[], + }; +} + +function readCachedManifest(value: unknown): PluginManifestListing | null { + if (!isObject(value)) return null; + const path = readString(value.path); + const id = readString(value.id); + const name = readString(value.name); + const version = readString(value.version); + const minHerdrVersion = readString(value.minHerdrVersion); + const description = value.description === null ? null : readString(value.description); + const platforms = value.platforms === null ? null : readStringArrayOrNull(value.platforms); + if ( + !path || + !isManifestPath(path) || + !id || + !name || + !version || + !minHerdrVersion || + (value.description !== null && !description) || + (value.platforms !== null && platforms === null) + ) { + return null; + } + return { path, id, name, version, minHerdrVersion, description, platforms }; +} + +export function normalizeRepositories(repositories: GitHubRepository[]): RepositoryListing[] { + const byId = new Map(); + for (const repository of repositories.map(normalizeRepository)) { + if (repository && !byId.has(repository.id)) byId.set(repository.id, repository); + } + return [...byId.values()].sort(compareRepositories); +} + +async function readBlacklistedRepositories(env: Env): Promise> { + const kv = env.PLUGIN_MARKETPLACE_BLACKLIST; + const blockedRepositories = new Set(); + if (!kv) return blockedRepositories; + + let cursor: string | undefined; + do { + const page = await kv.list({ prefix: BLACKLIST_REPO_KEY_PREFIX, cursor }); + for (const key of page.keys) { + const repository = key.name.slice(BLACKLIST_REPO_KEY_PREFIX.length).trim().toLowerCase(); + if (repository.includes("/")) blockedRepositories.add(repository); + } + cursor = page.cursor; + } while (cursor); + return blockedRepositories; +} + +function normalizeRepository(repo: GitHubRepository): RepositoryListing | null { + if ( + readBoolean(repo.disabled) || + readBoolean(repo.archived) || + readBoolean(repo.fork) || + readBoolean(repo.private) || + readString(repo.visibility) === "private" + ) { + return null; + } + + const fullNameParts = splitFullName(readString(repo.full_name)); + const ownerObject = isObject(repo.owner) ? repo.owner : {}; + const owner = firstString(readString(ownerObject.login), fullNameParts.owner); + const name = firstString(readString(repo.name), fullNameParts.name); + const defaultBranch = readString(repo.default_branch); + if (!owner || !name || !defaultBranch) return null; + + const fullName = readString(repo.full_name) ?? `${owner}/${name}`; + const url = readString(repo.html_url); + const id = readInteger(repo.id); + if (!url || !isValidGitHubRepoUrl(url, owner, name) || id === null || id <= 0) return null; + + return { + id, + fullName, + owner, + name, + description: readNullableString(repo.description), + url, + stars: readNonNegativeInteger(repo.stargazers_count), + forks: readNonNegativeInteger(repo.forks_count), + openIssues: readNonNegativeInteger(repo.open_issues_count), + language: readNullableString(repo.language), + topics: readStringArray(repo.topics), + createdAt: readIsoString(repo.created_at), + updatedAt: readIsoString(repo.updated_at), + pushedAt: readIsoString(repo.pushed_at), + defaultBranch, + }; +} + +function normalizeTreeEntry(value: unknown): GitHubTreeEntry | null { + if (!isObject(value)) return null; + const path = readString(value.path); + const mode = readString(value.mode); + const type = readString(value.type); + if (!path || !mode || !type) return null; + return { path, mode, type, size: readNonNegativeIntegerOrNull(value.size) }; +} + +function isManifestPath(path: string): boolean { + return path === PLUGIN_MANIFEST_FILE || path.endsWith(`/${PLUGIN_MANIFEST_FILE}`); +} + +function isDiscoverableManifestPath(path: string): boolean { + if (!isManifestPath(path)) return false; + const ignoredSegments = new Set([ + "test", + "tests", + "fixture", + "fixtures", + "testdata", + "__fixtures__", + ]); + return !path + .split("/") + .slice(0, -1) + .some((segment) => ignoredSegments.has(segment.toLowerCase())); +} + +function deduplicateManifests(manifests: PluginManifestListing[]): { + manifests: PluginManifestListing[]; + duplicateCount: number; +} { + const preferred = [...manifests].sort((a, b) => { + const depth = (path: string) => path.split("/").length; + return depth(a.path) - depth(b.path) || a.path.localeCompare(b.path); + }); + const byId = new Map(); + let duplicateCount = 0; + for (const manifest of preferred) { + if (byId.has(manifest.id)) { + duplicateCount += 1; + } else { + byId.set(manifest.id, manifest); + } + } + return { + manifests: [...byId.values()].sort((a, b) => a.path.localeCompare(b.path)), + duplicateCount, + }; +} + +function manifestCandidateKey(candidate: TreeManifestCandidate): string { + return `${candidate.repository.id}:${candidate.path}`; +} + +function compareRepositories(a: RepositoryListing, b: RepositoryListing): number { + return ( + b.stars - a.stars || + dateMs(b.pushedAt) - dateMs(a.pushedAt) || + a.fullName.localeCompare(b.fullName) + ); +} + +function githubRequestInit(token: string): RequestInit { + return { + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "User-Agent": "herdr-plugin-marketplace", + "X-GitHub-Api-Version": GITHUB_API_VERSION, + }, + }; +} + async function fetchWithTimeout( fetchFn: FetchLike, url: URL, @@ -226,91 +948,37 @@ async function fetchWithTimeout( } } -export function normalizeRepositories(repositories: GitHubRepository[]): PluginListing[] { - return repositories - .map(normalizeRepository) - .filter((plugin): plugin is PluginListing => plugin !== null) - .sort(comparePlugins); +function chunk(values: T[], size: number): T[][] { + const chunks: T[][] = []; + for (let index = 0; index < values.length; index += size) { + chunks.push(values.slice(index, index + size)); + } + return chunks; } -async function readBlacklistedRepositories(env: Env): Promise> { - const kv = env.PLUGIN_MARKETPLACE_BLACKLIST; - const blockedRepositories = new Set(); - if (!kv) { - return blockedRepositories; - } - - let cursor: string | undefined; - do { - const page = await kv.list({ prefix: BLACKLIST_REPO_KEY_PREFIX, cursor }); - for (const key of page.keys) { - const repository = key.name.slice(BLACKLIST_REPO_KEY_PREFIX.length).trim().toLowerCase(); - if (repository.includes("/")) { - blockedRepositories.add(repository); - } +async function mapConcurrent( + values: T[], + concurrency: number, + operation: (value: T) => Promise, +): Promise { + const results = new Array(values.length); + let nextIndex = 0; + const workers = Array.from({ length: Math.min(concurrency, values.length) }, async () => { + while (nextIndex < values.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await operation(values[index]); } - cursor = page.cursor; - } while (cursor); - - return blockedRepositories; -} - -function normalizeRepository(repo: GitHubRepository): PluginListing | null { - if ( - readBoolean(repo.disabled) || - readBoolean(repo.archived) || - readBoolean(repo.fork) || - readBoolean(repo.private) || - readString(repo.visibility) === "private" - ) { - return null; - } - - const fullNameParts = splitFullName(readString(repo.full_name)); - const ownerObject = isObject(repo.owner) ? repo.owner : {}; - const owner = firstString(readString(ownerObject.login), fullNameParts.owner); - const name = firstString(readString(repo.name), fullNameParts.name); - if (!owner || !name) { - return null; - } - - const fullName = readString(repo.full_name) ?? `${owner}/${name}`; - const url = readString(repo.html_url); - if (!url || !isValidGitHubRepoUrl(url, owner, name)) { - return null; - } - - return { - id: readInteger(repo.id) ?? 0, - fullName, - owner, - name, - description: readNullableString(repo.description), - url, - stars: readNonNegativeInteger(repo.stargazers_count), - forks: readNonNegativeInteger(repo.forks_count), - openIssues: readNonNegativeInteger(repo.open_issues_count), - language: readNullableString(repo.language), - topics: readStringArray(repo.topics), - createdAt: readIsoString(repo.created_at), - updatedAt: readIsoString(repo.updated_at), - pushedAt: readIsoString(repo.pushed_at), - }; -} - -function comparePlugins(a: PluginListing, b: PluginListing): number { - return ( - b.stars - a.stars || - dateMs(b.pushedAt) - dateMs(a.pushedAt) || - a.fullName.localeCompare(b.fullName) - ); + }); + await Promise.all(workers); + return results; } function jsonResponse(body: unknown, status: number, cacheControl: string): Response { return new Response(JSON.stringify(body), { status, headers: { - "Content-Type": "application/json; charset=utf-8", + "Content-Type": "application/json", "Cache-Control": cacheControl, }, }); @@ -333,14 +1001,9 @@ function isValidGitHubRepoUrl(url: string, owner: string, name: string): boolean } function splitFullName(fullName: string | null): { owner: string | null; name: string | null } { - if (!fullName) { - return { owner: null, name: null }; - } + if (!fullName) return { owner: null, name: null }; const [owner, name, extra] = fullName.split("/"); - if (!owner || !name || extra) { - return { owner: null, name: null }; - } - return { owner, name }; + return owner && name && !extra ? { owner, name } : { owner: null, name: null }; } function firstString(...values: Array): string | null { @@ -351,6 +1014,10 @@ function readString(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } +function readTrimmedString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + function readNullableString(value: unknown): string | null { return typeof value === "string" ? value : null; } @@ -359,13 +1026,23 @@ function readStringArray(value: unknown): string[] { return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; } +function readStringArrayOrNull(value: unknown): string[] | null { + return Array.isArray(value) && value.every((item) => typeof item === "string") + ? value + : null; +} + function readInteger(value: unknown): number | null { return typeof value === "number" && Number.isInteger(value) ? value : null; } function readNonNegativeInteger(value: unknown): number { + return readNonNegativeIntegerOrNull(value) ?? 0; +} + +function readNonNegativeIntegerOrNull(value: unknown): number | null { const integer = readInteger(value); - return integer !== null && integer >= 0 ? integer : 0; + return integer !== null && integer >= 0 ? integer : null; } function readBoolean(value: unknown): boolean { @@ -373,10 +1050,16 @@ function readBoolean(value: unknown): boolean { } function readIsoString(value: unknown): string | null { - if (typeof value !== "string") { - return null; - } - return Number.isNaN(Date.parse(value)) ? null : value; + return typeof value === "string" && !Number.isNaN(Date.parse(value)) ? value : null; +} + +function isHerdrVersion(value: string): boolean { + const normalized = value.startsWith("v") ? value.slice(1) : value; + const parts = normalized.split("."); + return ( + parts.length === 3 && + parts.every((part) => /^\d+$/.test(part) && Number(part) <= 0xffff_ffff) + ); } function dateMs(value: string | null): number { diff --git a/workers/plugin-marketplace/wrangler.toml b/workers/plugin-marketplace/wrangler.toml index d80c2bfb..76f118d7 100644 --- a/workers/plugin-marketplace/wrangler.toml +++ b/workers/plugin-marketplace/wrangler.toml @@ -3,6 +3,9 @@ main = "src/index.ts" compatibility_date = "2026-06-21" workers_dev = false +[limits] +subrequests = 3500 + [[r2_buckets]] binding = "PLUGIN_MARKETPLACE_BUCKET" bucket_name = "herdr-plugin-marketplace"