fix(release): repair agent mirror synchronization
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
import { basename, join } from "node:path";
|
||||
import { readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { spawn } from "node:child_process";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const DEFAULT_API_BASE = "https://api.atomgit.com/api/v5";
|
||||
const DEFAULT_REPOSITORY = "t8y2/dbx";
|
||||
@@ -62,7 +63,8 @@ async function main() {
|
||||
|
||||
const client = new AtomGitClient({ apiBase: args.apiBase, token: args.token, owner, repo });
|
||||
const atomRelease = await ensureRelease(client, release);
|
||||
const existingAssets = atomGitAssetNames(atomRelease);
|
||||
// PATCH responses may omit attachments, so re-read before deciding what must be replaced.
|
||||
const existingAssets = atomGitAssets((await client.getRelease(tag)) || atomRelease);
|
||||
const assets = localAssets(args.assetsDir);
|
||||
if (assets.length === 0) {
|
||||
console.log("No release assets found to sync.");
|
||||
@@ -78,8 +80,28 @@ async function main() {
|
||||
return true;
|
||||
});
|
||||
|
||||
if (!args.skipExistingAssets) {
|
||||
for (const assetPath of pendingAssets) {
|
||||
const name = basename(assetPath);
|
||||
const existingAsset = existingAssets.get(name);
|
||||
if (!existingAsset) continue;
|
||||
if (existingAsset.id === null) {
|
||||
throw new Error(`Cannot replace AtomGit asset without an attachment id: ${name}`);
|
||||
}
|
||||
console.log(`Deleting existing AtomGit asset: ${name}`);
|
||||
await client.deleteAsset(tag, existingAsset.id);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Uploading ${pendingAssets.length} asset(s) with concurrency ${args.concurrency}.`);
|
||||
await mapWithConcurrency(pendingAssets, args.concurrency, (assetPath) => uploadAsset(client, tag, assetPath));
|
||||
|
||||
const refreshedRelease = await client.getRelease(tag);
|
||||
const uploadedAssets = atomGitAssets(refreshedRelease);
|
||||
const missingAssets = pendingAssets.map(basename).filter((name) => !uploadedAssets.has(name));
|
||||
if (missingAssets.length > 0) {
|
||||
throw new Error(`AtomGit did not register uploaded assets: ${missingAssets.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureRelease(client, githubRelease) {
|
||||
@@ -121,9 +143,10 @@ async function uploadAsset(client, tag, filePath) {
|
||||
|
||||
// AtomGit returns a per-file upload URL plus required upload headers.
|
||||
// Try raw PUT first and fall back to multipart POST for API-compatible deployments.
|
||||
const putStatus = await runCurl([
|
||||
const putResult = await runCurl([
|
||||
"--fail-with-body",
|
||||
"--location",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--retry",
|
||||
"3",
|
||||
@@ -138,14 +161,15 @@ async function uploadAsset(client, tag, filePath) {
|
||||
filePath,
|
||||
uploadTarget.url,
|
||||
]);
|
||||
if (putStatus === 0) {
|
||||
if (uploadResultSucceeded(putResult)) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.warn(`PUT upload failed for ${name}; retrying as multipart POST.`);
|
||||
const postStatus = await runCurl([
|
||||
console.warn(`PUT upload failed for ${name}: ${uploadFailureMessage(putResult)}; retrying as multipart POST.`);
|
||||
const postResult = await runCurl([
|
||||
"--fail-with-body",
|
||||
"--location",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--retry",
|
||||
"3",
|
||||
@@ -158,19 +182,45 @@ async function uploadAsset(client, tag, filePath) {
|
||||
`file=@${filePath};filename=${name};type=${contentTypeHeader}`,
|
||||
uploadTarget.url,
|
||||
]);
|
||||
if (postStatus !== 0) {
|
||||
throw new Error(`Failed to upload ${name} to AtomGit.`);
|
||||
if (!uploadResultSucceeded(postResult)) {
|
||||
throw new Error(`Failed to upload ${name} to AtomGit: ${uploadFailureMessage(postResult)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function runCurl(args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("curl", args, { stdio: "inherit" });
|
||||
const child = spawn("curl", args, { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.once("error", reject);
|
||||
child.once("close", (code) => resolve(code ?? 1));
|
||||
child.once("close", (code) => resolve({ status: code ?? 1, stdout, stderr }));
|
||||
});
|
||||
}
|
||||
|
||||
export function uploadResultSucceeded(result) {
|
||||
if (result.status !== 0) return false;
|
||||
const responseBody = result.stdout.trim();
|
||||
if (!responseBody || responseBody.toLowerCase() === "success") return true;
|
||||
try {
|
||||
const response = JSON.parse(responseBody);
|
||||
return response.success === true || response.status === "success";
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
function uploadFailureMessage(result) {
|
||||
return result.stderr.trim() || result.stdout.trim() || `curl exited with status ${result.status}`;
|
||||
}
|
||||
|
||||
async function mapWithConcurrency(items, concurrency, worker) {
|
||||
let nextIndex = 0;
|
||||
async function runWorker() {
|
||||
@@ -189,8 +239,8 @@ function localAssets(dir) {
|
||||
.sort((a, b) => statSync(a).size - statSync(b).size || basename(a).localeCompare(basename(b)));
|
||||
}
|
||||
|
||||
function atomGitAssetNames(release) {
|
||||
const names = new Set();
|
||||
export function atomGitAssets(release) {
|
||||
const assets = new Map();
|
||||
const groups = [
|
||||
release?.assets,
|
||||
release?.attach_files,
|
||||
@@ -205,11 +255,14 @@ function atomGitAssetNames(release) {
|
||||
for (const item of group) {
|
||||
const name = item?.name || item?.file_name || item?.filename || item?.title;
|
||||
if (name) {
|
||||
names.add(name);
|
||||
assets.set(name, {
|
||||
id: Number.isInteger(item?.id) ? item.id : null,
|
||||
name,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
return assets;
|
||||
}
|
||||
|
||||
function contentTypeFor(name) {
|
||||
@@ -230,7 +283,7 @@ function headerValue(headers, name) {
|
||||
return entry ? entry[1] : "";
|
||||
}
|
||||
|
||||
class AtomGitClient {
|
||||
export class AtomGitClient {
|
||||
constructor({ apiBase, token, owner, repo }) {
|
||||
this.apiBase = apiBase.replace(/\/+$/, "");
|
||||
this.token = token;
|
||||
@@ -284,6 +337,13 @@ class AtomGitClient {
|
||||
};
|
||||
}
|
||||
|
||||
async deleteAsset(tag, assetId) {
|
||||
await this.request(
|
||||
"DELETE",
|
||||
`/repos/${this.owner}/${this.repo}/releases/${encodeURIComponent(tag)}/attach_files/${assetId}`,
|
||||
);
|
||||
}
|
||||
|
||||
async request(method, path, body = null, requestOptions = {}) {
|
||||
const headers = {
|
||||
Accept: "application/json",
|
||||
@@ -337,7 +397,9 @@ function sanitizeUrlForLog(value) {
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer } from "node:http";
|
||||
import test from "node:test";
|
||||
|
||||
import { AtomGitClient, atomGitAssets, uploadResultSucceeded } from "./sync-atomgit-release.mjs";
|
||||
|
||||
test("uploadResultSucceeded rejects AtomGit callback errors returned with HTTP success", () => {
|
||||
assert.equal(uploadResultSucceeded({ status: 0, stdout: "success", stderr: "" }), true);
|
||||
assert.equal(
|
||||
uploadResultSucceeded({
|
||||
status: 0,
|
||||
stdout: JSON.stringify({ message: "Fail to read response body", code: "CallBack.0002" }),
|
||||
stderr: "",
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("atomGitAssets preserves attachment ids required for replacement", () => {
|
||||
assert.deepEqual(
|
||||
atomGitAssets({ assets: [{ id: 118266, name: "agent-registry.json" }] }),
|
||||
new Map([["agent-registry.json", { id: 118266, name: "agent-registry.json" }]]),
|
||||
);
|
||||
});
|
||||
|
||||
test("deleteAsset removes an existing release attachment by id", async (t) => {
|
||||
const requests = [];
|
||||
const server = createServer((request, response) => {
|
||||
requests.push({ method: request.method, url: request.url });
|
||||
response.statusCode = 204;
|
||||
response.end();
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
t.after(() => new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))));
|
||||
|
||||
const address = server.address();
|
||||
assert.ok(address && typeof address !== "string");
|
||||
const client = new AtomGitClient({
|
||||
apiBase: `http://127.0.0.1:${address.port}`,
|
||||
token: "test-token",
|
||||
owner: "t8y2",
|
||||
repo: "dbx",
|
||||
});
|
||||
|
||||
await client.deleteAsset("agents-latest", 118266);
|
||||
assert.deepEqual(requests, [
|
||||
{
|
||||
method: "DELETE",
|
||||
url: "/repos/t8y2/dbx/releases/agents-latest/attach_files/118266",
|
||||
},
|
||||
]);
|
||||
});
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import { createReadStream, readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { basename, join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const DEFAULT_API_BASE = "https://api.cnb.cool";
|
||||
const DEFAULT_REPOSITORY = "dbxio.com/dbx";
|
||||
@@ -76,7 +77,7 @@ async function uploadWithRetry(client, releaseId, filePath, overwriteExisting) {
|
||||
}
|
||||
}
|
||||
|
||||
class CnbClient {
|
||||
export class CnbClient {
|
||||
constructor({ apiBase, repository, token }) {
|
||||
this.apiBase = apiBase.replace(/\/+$/, "");
|
||||
this.repository = repository;
|
||||
@@ -135,7 +136,14 @@ class CnbClient {
|
||||
});
|
||||
if (allow404 && response.status === 404) return null;
|
||||
if (!response.ok) throw new Error(`CNB API ${method} ${path} failed with ${response.status}: ${await response.text()}`);
|
||||
return response.status === 204 ? null : response.json();
|
||||
const responseBody = await response.text();
|
||||
// CNB may acknowledge release metadata updates with HTTP 200 and an empty body.
|
||||
if (!responseBody.trim()) return null;
|
||||
try {
|
||||
return JSON.parse(responseBody);
|
||||
} catch (error) {
|
||||
throw new Error(`CNB API ${method} ${path} returned invalid JSON: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
headers(json = false) {
|
||||
@@ -165,7 +173,9 @@ async function mapWithConcurrency(items, concurrency, worker) {
|
||||
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, runWorker));
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer } from "node:http";
|
||||
import test from "node:test";
|
||||
|
||||
import { CnbClient } from "./sync-cnb-release.mjs";
|
||||
|
||||
test("ensureRelease accepts an empty successful PATCH response", async (t) => {
|
||||
const release = { id: "release-1", tag_name: "agents-latest", assets: [] };
|
||||
const requests = [];
|
||||
const server = createServer((request, response) => {
|
||||
requests.push({ method: request.method, url: request.url });
|
||||
if (request.method === "GET") {
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(JSON.stringify(release));
|
||||
return;
|
||||
}
|
||||
if (request.method === "PATCH") {
|
||||
response.statusCode = 200;
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
response.statusCode = 500;
|
||||
response.end();
|
||||
});
|
||||
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
t.after(() => new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))));
|
||||
|
||||
const address = server.address();
|
||||
assert.ok(address && typeof address !== "string");
|
||||
const client = new CnbClient({
|
||||
apiBase: `http://127.0.0.1:${address.port}`,
|
||||
repository: "dbxio.com/dbx",
|
||||
token: "test-token",
|
||||
});
|
||||
|
||||
const result = await client.ensureRelease("agents-latest", {
|
||||
name: "Latest agents",
|
||||
body: "Latest stable agent release",
|
||||
});
|
||||
|
||||
assert.deepEqual(result, release);
|
||||
assert.deepEqual(requests, [
|
||||
{ method: "GET", url: "/dbxio.com/dbx/-/releases/tags/agents-latest" },
|
||||
{ method: "PATCH", url: "/dbxio.com/dbx/-/releases/release-1" },
|
||||
]);
|
||||
});
|
||||
Reference in New Issue
Block a user