From 71497f0851a1a85da3085805cd13f1d0bbf2ad2f Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Thu, 16 Jul 2026 10:02:37 +0800 Subject: [PATCH] fix(release): repair agent mirror synchronization --- .github/scripts/sync-atomgit-release.mjs | 98 +++++++++++++++---- .github/scripts/sync-atomgit-release.test.mjs | 52 ++++++++++ .github/scripts/sync-cnb-release.mjs | 22 +++-- .github/scripts/sync-cnb-release.test.mjs | 47 +++++++++ crates/dbx-core/src/lib.rs | 60 +++++++++--- 5 files changed, 244 insertions(+), 35 deletions(-) create mode 100644 .github/scripts/sync-atomgit-release.test.mjs create mode 100644 .github/scripts/sync-cnb-release.test.mjs diff --git a/.github/scripts/sync-atomgit-release.mjs b/.github/scripts/sync-atomgit-release.mjs index e7be13e81..489b4e174 100644 --- a/.github/scripts/sync-atomgit-release.mjs +++ b/.github/scripts/sync-atomgit-release.mjs @@ -3,6 +3,7 @@ import { basename, join } from "node:path"; import { readdirSync, readFileSync, statSync } from "node:fs"; import { spawn } from "node:child_process"; +import { pathToFileURL } from "node:url"; const DEFAULT_API_BASE = "https://api.atomgit.com/api/v5"; const DEFAULT_REPOSITORY = "t8y2/dbx"; @@ -62,7 +63,8 @@ async function main() { const client = new AtomGitClient({ apiBase: args.apiBase, token: args.token, owner, repo }); const atomRelease = await ensureRelease(client, release); - const existingAssets = atomGitAssetNames(atomRelease); + // PATCH responses may omit attachments, so re-read before deciding what must be replaced. + const existingAssets = atomGitAssets((await client.getRelease(tag)) || atomRelease); const assets = localAssets(args.assetsDir); if (assets.length === 0) { console.log("No release assets found to sync."); @@ -78,8 +80,28 @@ async function main() { return true; }); + if (!args.skipExistingAssets) { + for (const assetPath of pendingAssets) { + const name = basename(assetPath); + const existingAsset = existingAssets.get(name); + if (!existingAsset) continue; + if (existingAsset.id === null) { + throw new Error(`Cannot replace AtomGit asset without an attachment id: ${name}`); + } + console.log(`Deleting existing AtomGit asset: ${name}`); + await client.deleteAsset(tag, existingAsset.id); + } + } + console.log(`Uploading ${pendingAssets.length} asset(s) with concurrency ${args.concurrency}.`); await mapWithConcurrency(pendingAssets, args.concurrency, (assetPath) => uploadAsset(client, tag, assetPath)); + + const refreshedRelease = await client.getRelease(tag); + const uploadedAssets = atomGitAssets(refreshedRelease); + const missingAssets = pendingAssets.map(basename).filter((name) => !uploadedAssets.has(name)); + if (missingAssets.length > 0) { + throw new Error(`AtomGit did not register uploaded assets: ${missingAssets.join(", ")}`); + } } async function ensureRelease(client, githubRelease) { @@ -121,9 +143,10 @@ async function uploadAsset(client, tag, filePath) { // AtomGit returns a per-file upload URL plus required upload headers. // Try raw PUT first and fall back to multipart POST for API-compatible deployments. - const putStatus = await runCurl([ + const putResult = await runCurl([ "--fail-with-body", "--location", + "--silent", "--show-error", "--retry", "3", @@ -138,14 +161,15 @@ async function uploadAsset(client, tag, filePath) { filePath, uploadTarget.url, ]); - if (putStatus === 0) { + if (uploadResultSucceeded(putResult)) { return; } - console.warn(`PUT upload failed for ${name}; retrying as multipart POST.`); - const postStatus = await runCurl([ + console.warn(`PUT upload failed for ${name}: ${uploadFailureMessage(putResult)}; retrying as multipart POST.`); + const postResult = await runCurl([ "--fail-with-body", "--location", + "--silent", "--show-error", "--retry", "3", @@ -158,19 +182,45 @@ async function uploadAsset(client, tag, filePath) { `file=@${filePath};filename=${name};type=${contentTypeHeader}`, uploadTarget.url, ]); - if (postStatus !== 0) { - throw new Error(`Failed to upload ${name} to AtomGit.`); + if (!uploadResultSucceeded(postResult)) { + throw new Error(`Failed to upload ${name} to AtomGit: ${uploadFailureMessage(postResult)}`); } } function runCurl(args) { return new Promise((resolve, reject) => { - const child = spawn("curl", args, { stdio: "inherit" }); + const child = spawn("curl", args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); child.once("error", reject); - child.once("close", (code) => resolve(code ?? 1)); + child.once("close", (code) => resolve({ status: code ?? 1, stdout, stderr })); }); } +export function uploadResultSucceeded(result) { + if (result.status !== 0) return false; + const responseBody = result.stdout.trim(); + if (!responseBody || responseBody.toLowerCase() === "success") return true; + try { + const response = JSON.parse(responseBody); + return response.success === true || response.status === "success"; + } catch { + return true; + } +} + +function uploadFailureMessage(result) { + return result.stderr.trim() || result.stdout.trim() || `curl exited with status ${result.status}`; +} + async function mapWithConcurrency(items, concurrency, worker) { let nextIndex = 0; async function runWorker() { @@ -189,8 +239,8 @@ function localAssets(dir) { .sort((a, b) => statSync(a).size - statSync(b).size || basename(a).localeCompare(basename(b))); } -function atomGitAssetNames(release) { - const names = new Set(); +export function atomGitAssets(release) { + const assets = new Map(); const groups = [ release?.assets, release?.attach_files, @@ -205,11 +255,14 @@ function atomGitAssetNames(release) { for (const item of group) { const name = item?.name || item?.file_name || item?.filename || item?.title; if (name) { - names.add(name); + assets.set(name, { + id: Number.isInteger(item?.id) ? item.id : null, + name, + }); } } } - return names; + return assets; } function contentTypeFor(name) { @@ -230,7 +283,7 @@ function headerValue(headers, name) { return entry ? entry[1] : ""; } -class AtomGitClient { +export class AtomGitClient { constructor({ apiBase, token, owner, repo }) { this.apiBase = apiBase.replace(/\/+$/, ""); this.token = token; @@ -284,6 +337,13 @@ class AtomGitClient { }; } + async deleteAsset(tag, assetId) { + await this.request( + "DELETE", + `/repos/${this.owner}/${this.repo}/releases/${encodeURIComponent(tag)}/attach_files/${assetId}`, + ); + } + async request(method, path, body = null, requestOptions = {}) { const headers = { Accept: "application/json", @@ -337,7 +397,9 @@ function sanitizeUrlForLog(value) { } } -main().catch((error) => { - console.error(error); - process.exit(1); -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.github/scripts/sync-atomgit-release.test.mjs b/.github/scripts/sync-atomgit-release.test.mjs new file mode 100644 index 000000000..048d3995b --- /dev/null +++ b/.github/scripts/sync-atomgit-release.test.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import test from "node:test"; + +import { AtomGitClient, atomGitAssets, uploadResultSucceeded } from "./sync-atomgit-release.mjs"; + +test("uploadResultSucceeded rejects AtomGit callback errors returned with HTTP success", () => { + assert.equal(uploadResultSucceeded({ status: 0, stdout: "success", stderr: "" }), true); + assert.equal( + uploadResultSucceeded({ + status: 0, + stdout: JSON.stringify({ message: "Fail to read response body", code: "CallBack.0002" }), + stderr: "", + }), + false, + ); +}); + +test("atomGitAssets preserves attachment ids required for replacement", () => { + assert.deepEqual( + atomGitAssets({ assets: [{ id: 118266, name: "agent-registry.json" }] }), + new Map([["agent-registry.json", { id: 118266, name: "agent-registry.json" }]]), + ); +}); + +test("deleteAsset removes an existing release attachment by id", async (t) => { + const requests = []; + const server = createServer((request, response) => { + requests.push({ method: request.method, url: request.url }); + response.statusCode = 204; + response.end(); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())))); + + const address = server.address(); + assert.ok(address && typeof address !== "string"); + const client = new AtomGitClient({ + apiBase: `http://127.0.0.1:${address.port}`, + token: "test-token", + owner: "t8y2", + repo: "dbx", + }); + + await client.deleteAsset("agents-latest", 118266); + assert.deepEqual(requests, [ + { + method: "DELETE", + url: "/repos/t8y2/dbx/releases/agents-latest/attach_files/118266", + }, + ]); +}); diff --git a/.github/scripts/sync-cnb-release.mjs b/.github/scripts/sync-cnb-release.mjs index c3f080966..10893ac87 100644 --- a/.github/scripts/sync-cnb-release.mjs +++ b/.github/scripts/sync-cnb-release.mjs @@ -2,6 +2,7 @@ import { createReadStream, readdirSync, readFileSync, statSync } from "node:fs"; import { basename, join } from "node:path"; +import { pathToFileURL } from "node:url"; const DEFAULT_API_BASE = "https://api.cnb.cool"; const DEFAULT_REPOSITORY = "dbxio.com/dbx"; @@ -76,7 +77,7 @@ async function uploadWithRetry(client, releaseId, filePath, overwriteExisting) { } } -class CnbClient { +export class CnbClient { constructor({ apiBase, repository, token }) { this.apiBase = apiBase.replace(/\/+$/, ""); this.repository = repository; @@ -135,7 +136,14 @@ class CnbClient { }); if (allow404 && response.status === 404) return null; if (!response.ok) throw new Error(`CNB API ${method} ${path} failed with ${response.status}: ${await response.text()}`); - return response.status === 204 ? null : response.json(); + const responseBody = await response.text(); + // CNB may acknowledge release metadata updates with HTTP 200 and an empty body. + if (!responseBody.trim()) return null; + try { + return JSON.parse(responseBody); + } catch (error) { + throw new Error(`CNB API ${method} ${path} returned invalid JSON: ${error.message}`); + } } headers(json = false) { @@ -165,7 +173,9 @@ async function mapWithConcurrency(items, concurrency, worker) { await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, runWorker)); } -main().catch((error) => { - console.error(error); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/.github/scripts/sync-cnb-release.test.mjs b/.github/scripts/sync-cnb-release.test.mjs new file mode 100644 index 000000000..9d2a2d799 --- /dev/null +++ b/.github/scripts/sync-cnb-release.test.mjs @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import test from "node:test"; + +import { CnbClient } from "./sync-cnb-release.mjs"; + +test("ensureRelease accepts an empty successful PATCH response", async (t) => { + const release = { id: "release-1", tag_name: "agents-latest", assets: [] }; + const requests = []; + const server = createServer((request, response) => { + requests.push({ method: request.method, url: request.url }); + if (request.method === "GET") { + response.setHeader("Content-Type", "application/json"); + response.end(JSON.stringify(release)); + return; + } + if (request.method === "PATCH") { + response.statusCode = 200; + response.end(); + return; + } + response.statusCode = 500; + response.end(); + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())))); + + const address = server.address(); + assert.ok(address && typeof address !== "string"); + const client = new CnbClient({ + apiBase: `http://127.0.0.1:${address.port}`, + repository: "dbxio.com/dbx", + token: "test-token", + }); + + const result = await client.ensureRelease("agents-latest", { + name: "Latest agents", + body: "Latest stable agent release", + }); + + assert.deepEqual(result, release); + assert.deepEqual(requests, [ + { method: "GET", url: "/dbxio.com/dbx/-/releases/tags/agents-latest" }, + { method: "PATCH", url: "/dbxio.com/dbx/-/releases/release-1" }, + ]); +}); diff --git a/crates/dbx-core/src/lib.rs b/crates/dbx-core/src/lib.rs index 99bc3a58a..3a42d3655 100644 --- a/crates/dbx-core/src/lib.rs +++ b/crates/dbx-core/src/lib.rs @@ -71,7 +71,7 @@ pub mod xlsx_export; pub const R2_CDN_BASE: &str = "https://dl.dbxio.com/"; pub const GITHUB_RELEASE_DOWNLOAD_PREFIX: &str = "https://github.com/t8y2/dbx/releases/download/"; pub const CNB_RELEASE_DOWNLOAD_PREFIX: &str = "https://cnb.cool/dbxio.com/dbx/-/releases/download/"; -pub const ATOMGIT_RELEASE_DOWNLOAD_PREFIX: &str = "https://atomgit.com/t8y2/dbx/releases/download/"; +pub const ATOMGIT_RELEASE_API_PREFIX: &str = "https://api.atomgit.com/api/v5/repos/t8y2/dbx/releases/"; #[derive(Clone, Copy, Debug, Default, serde::Deserialize, PartialEq, Eq, Hash)] #[serde(rename_all = "lowercase")] @@ -86,18 +86,30 @@ impl DownloadSource { pub fn download_candidate_urls(self, github_url: &str, r2_path: &str) -> Result, String> { match self { Self::Official => Ok(download_candidate_urls(github_url, r2_path)), - Self::Cnb => Ok(vec![ + Self::Cnb => Ok(mirror_download_candidate_urls( + github_url, + r2_path, rewrite_github_release_url(github_url, CNB_RELEASE_DOWNLOAD_PREFIX)?, - format!("{R2_CDN_BASE}{r2_path}"), - ]), - Self::Atomgit => Ok(vec![ - rewrite_github_release_url(github_url, ATOMGIT_RELEASE_DOWNLOAD_PREFIX)?, - format!("{R2_CDN_BASE}{r2_path}"), - ]), + )), + Self::Atomgit => Ok(mirror_download_candidate_urls( + github_url, + r2_path, + rewrite_github_release_url_for_atomgit(github_url)?, + )), } } } +fn mirror_download_candidate_urls(github_url: &str, r2_path: &str, mirror_url: String) -> Vec { + let r2_url = format!("{R2_CDN_BASE}{r2_path}"); + // Mutable mirror aliases can lag even when versioned release assets are healthy. + if github_url.ends_with("/agents-latest/agent-registry.json") { + vec![r2_url, mirror_url] + } else { + vec![mirror_url, r2_url] + } +} + fn rewrite_github_release_url(url: &str, target_prefix: &str) -> Result { if url.starts_with(target_prefix) { return Ok(url.to_string()); @@ -107,6 +119,20 @@ fn rewrite_github_release_url(url: &str, target_prefix: &str) -> Result Result { + if url.starts_with(ATOMGIT_RELEASE_API_PREFIX) { + return Ok(url.to_string()); + } + let path = url + .strip_prefix(GITHUB_RELEASE_DOWNLOAD_PREFIX) + .ok_or_else(|| format!("Unsupported DBX release download URL: {url}"))?; + let (tag, file_name) = path + .split_once('/') + .filter(|(_, file_name)| !file_name.contains('/')) + .ok_or_else(|| format!("Unsupported DBX release asset path: {path}"))?; + Ok(format!("{ATOMGIT_RELEASE_API_PREFIX}{tag}/attach_files/{file_name}/download")) +} + pub fn download_candidate_urls(github_url: &str, r2_path: &str) -> Vec { vec![format!("{R2_CDN_BASE}{r2_path}"), github_url.to_string()] } @@ -168,20 +194,32 @@ mod tests { } #[test] - fn mirror_download_candidates_rewrite_release_urls() { + fn mirror_download_candidates_prefer_stable_registry_metadata() { let github_url = "https://github.com/t8y2/dbx/releases/download/agents-latest/agent-registry.json"; assert_eq!( DownloadSource::Cnb.download_candidate_urls(github_url, "agents/agent-registry.json").unwrap(), vec![ - "https://cnb.cool/dbxio.com/dbx/-/releases/download/agents-latest/agent-registry.json", "https://dl.dbxio.com/agents/agent-registry.json", + "https://cnb.cool/dbxio.com/dbx/-/releases/download/agents-latest/agent-registry.json", ] ); assert_eq!( DownloadSource::Atomgit.download_candidate_urls(github_url, "agents/agent-registry.json").unwrap(), vec![ - "https://atomgit.com/t8y2/dbx/releases/download/agents-latest/agent-registry.json", "https://dl.dbxio.com/agents/agent-registry.json", + "https://api.atomgit.com/api/v5/repos/t8y2/dbx/releases/agents-latest/attach_files/agent-registry.json/download", + ] + ); + } + + #[test] + fn atomgit_versioned_assets_use_attachment_download_api_first() { + let github_url = "https://github.com/t8y2/dbx/releases/download/agents-v0.2.55/dbx-agent-h2.jar"; + assert_eq!( + DownloadSource::Atomgit.download_candidate_urls(github_url, "agents/dbx-agent-h2.jar").unwrap(), + vec![ + "https://api.atomgit.com/api/v5/repos/t8y2/dbx/releases/agents-v0.2.55/attach_files/dbx-agent-h2.jar/download", + "https://dl.dbxio.com/agents/dbx-agent-h2.jar", ] ); }