diff --git a/.github/scripts/sync-atomgit-release.mjs b/.github/scripts/sync-atomgit-release.mjs index 18bed4db6..e7be13e81 100644 --- a/.github/scripts/sync-atomgit-release.mjs +++ b/.github/scripts/sync-atomgit-release.mjs @@ -2,10 +2,11 @@ import { basename, join } from "node:path"; import { readdirSync, readFileSync, statSync } from "node:fs"; -import { spawnSync } from "node:child_process"; +import { spawn } from "node:child_process"; const DEFAULT_API_BASE = "https://api.atomgit.com/api/v5"; const DEFAULT_REPOSITORY = "t8y2/dbx"; +const DEFAULT_CONCURRENCY = 3; function parseArgs(argv) { const args = { @@ -15,6 +16,7 @@ function parseArgs(argv) { githubReleasePath: "", assetsDir: "", skipExistingAssets: true, + concurrency: Number.parseInt(process.env.ATOMGIT_UPLOAD_CONCURRENCY || `${DEFAULT_CONCURRENCY}`, 10), }; for (let i = 0; i < argv.length; i++) { @@ -43,6 +45,9 @@ function parseArgs(argv) { if (!args.repository.includes("/")) { throw new Error(`Invalid AtomGit repository: ${args.repository}`); } + if (!Number.isInteger(args.concurrency) || args.concurrency < 1) { + throw new Error("ATOMGIT_UPLOAD_CONCURRENCY must be a positive integer."); + } return args; } @@ -64,15 +69,17 @@ async function main() { return; } - console.log(`Syncing ${assets.length} asset(s) to AtomGit release ${tag}.`); - for (const assetPath of assets) { + const pendingAssets = assets.filter((assetPath) => { const name = basename(assetPath); if (args.skipExistingAssets && existingAssets.has(name)) { console.log(`Skipping existing AtomGit asset: ${name}`); - continue; + return false; } - await uploadAsset(client, tag, assetPath); - } + return true; + }); + + console.log(`Uploading ${pendingAssets.length} asset(s) with concurrency ${args.concurrency}.`); + await mapWithConcurrency(pendingAssets, args.concurrency, (assetPath) => uploadAsset(client, tag, assetPath)); } async function ensureRelease(client, githubRelease) { @@ -114,7 +121,7 @@ async function uploadAsset(client, tag, filePath) { // AtomGit returns a per-file upload URL plus required upload headers. // Try raw PUT first and fall back to multipart POST for API-compatible deployments. - const put = runCurl([ + const putStatus = await runCurl([ "--fail-with-body", "--location", "--show-error", @@ -131,12 +138,12 @@ async function uploadAsset(client, tag, filePath) { filePath, uploadTarget.url, ]); - if (put.status === 0) { + if (putStatus === 0) { return; } console.warn(`PUT upload failed for ${name}; retrying as multipart POST.`); - const post = runCurl([ + const postStatus = await runCurl([ "--fail-with-body", "--location", "--show-error", @@ -151,13 +158,28 @@ async function uploadAsset(client, tag, filePath) { `file=@${filePath};filename=${name};type=${contentTypeHeader}`, uploadTarget.url, ]); - if (post.status !== 0) { + if (postStatus !== 0) { throw new Error(`Failed to upload ${name} to AtomGit.`); } } function runCurl(args) { - return spawnSync("curl", args, { stdio: "inherit" }); + return new Promise((resolve, reject) => { + const child = spawn("curl", args, { stdio: "inherit" }); + child.once("error", reject); + child.once("close", (code) => resolve(code ?? 1)); + }); +} + +async function mapWithConcurrency(items, concurrency, worker) { + let nextIndex = 0; + async function runWorker() { + while (nextIndex < items.length) { + const index = nextIndex++; + await worker(items[index]); + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, runWorker)); } function localAssets(dir) { diff --git a/.github/scripts/sync-cnb-release.mjs b/.github/scripts/sync-cnb-release.mjs new file mode 100644 index 000000000..52c355d8e --- /dev/null +++ b/.github/scripts/sync-cnb-release.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node + +import { createReadStream, readdirSync, readFileSync, statSync } from "node:fs"; +import { basename, join } from "node:path"; + +const DEFAULT_API_BASE = "https://api.cnb.cool"; +const DEFAULT_REPOSITORY = "dbxio.com/dbx"; +const DEFAULT_CONCURRENCY = 3; +const MAX_ATTEMPTS = 4; + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const githubRelease = JSON.parse(readFileSync(args.githubReleasePath, "utf8")); + const tag = githubRelease.tagName || githubRelease.tag_name; + if (!tag) throw new Error("GitHub release JSON is missing tagName."); + + const client = new CnbClient(args); + const release = await client.ensureRelease(tag, githubRelease); + const existingAssets = new Set((release.assets || []).map((asset) => asset.name)); + const assets = localAssets(args.assetsDir).filter((assetPath) => { + const name = basename(assetPath); + if (existingAssets.has(name)) { + console.log(`Skipping existing CNB asset: ${name}`); + return false; + } + return true; + }); + + console.log(`Uploading ${assets.length} CNB asset(s) with concurrency ${args.concurrency}.`); + await mapWithConcurrency(assets, args.concurrency, (assetPath) => uploadWithRetry(client, release.id, assetPath)); +} + +function parseArgs(argv) { + const args = { + apiBase: process.env.CNB_API_BASE || DEFAULT_API_BASE, + repository: process.env.CNB_REPOSITORY || DEFAULT_REPOSITORY, + token: process.env.CNB_TOKEN || "", + concurrency: Number.parseInt(process.env.CNB_UPLOAD_CONCURRENCY || `${DEFAULT_CONCURRENCY}`, 10), + githubReleasePath: "", + assetsDir: "", + }; + for (let index = 0; index < argv.length; index++) { + const arg = argv[index]; + if (arg === "--github-release") args.githubReleasePath = argv[++index]; + else if (arg === "--assets-dir") args.assetsDir = argv[++index]; + else throw new Error(`Unknown argument: ${arg}`); + } + if (!args.token) throw new Error("CNB_TOKEN is required."); + if (!args.githubReleasePath || !args.assetsDir) { + throw new Error("Usage: sync-cnb-release.mjs --github-release --assets-dir "); + } + if (!Number.isInteger(args.concurrency) || args.concurrency < 1) { + throw new Error("CNB_UPLOAD_CONCURRENCY must be a positive integer."); + } + return args; +} + +async function uploadWithRetry(client, releaseId, filePath) { + const name = basename(filePath); + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + console.log(`Uploading ${name}, attempt ${attempt}/${MAX_ATTEMPTS}.`); + await client.uploadAsset(releaseId, filePath); + console.log(`Uploaded ${name}.`); + return; + } catch (error) { + if (attempt === MAX_ATTEMPTS) throw error; + const delayMs = 2 ** (attempt - 1) * 5000; + console.warn(`Upload failed for ${name}: ${error.message}; retrying in ${delayMs / 1000}s.`); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } +} + +class CnbClient { + constructor({ apiBase, repository, token }) { + this.apiBase = apiBase.replace(/\/+$/, ""); + this.repository = repository; + this.token = token; + } + + async ensureRelease(tag, githubRelease) { + const payload = { + tag_name: tag, + name: githubRelease.name || tag, + body: githubRelease.body || "", + prerelease: Boolean(githubRelease.isPrerelease || githubRelease.prerelease), + target_commitish: githubRelease.targetCommitish || githubRelease.target_commitish || "", + }; + const existing = await this.request("GET", `/${this.repository}/-/releases/tags/${encodeURIComponent(tag)}`, null, true); + if (!existing) return this.request("POST", `/${this.repository}/-/releases`, payload); + + // Keep release metadata aligned while preserving the existing asset list used for resumable uploads. + await this.request("PATCH", `/${this.repository}/-/releases/${existing.id}`, { + name: payload.name, + body: payload.body, + }); + return existing; + } + + async uploadAsset(releaseId, filePath) { + const size = statSync(filePath).size; + const target = await this.request("POST", `/${this.repository}/-/releases/${releaseId}/asset-upload-url`, { + asset_name: basename(filePath), + size, + overwrite: false, + }); + const uploadResponse = await fetch(target.upload_url, { + method: "PUT", + headers: { "Content-Type": "application/octet-stream", "Content-Length": `${size}` }, + body: createReadStream(filePath), + duplex: "half", + }); + if (!uploadResponse.ok) { + throw new Error(`CNB object upload failed with ${uploadResponse.status}: ${await uploadResponse.text()}`); + } + const verifyResponse = await fetch(decodeURIComponent(target.verify_url), { + method: "POST", + headers: this.headers(), + }); + if (!verifyResponse.ok) { + throw new Error(`CNB upload confirmation failed with ${verifyResponse.status}: ${await verifyResponse.text()}`); + } + } + + async request(method, path, body = null, allow404 = false) { + const response = await fetch(`${this.apiBase}${path}`, { + method, + headers: this.headers(body !== null), + body: body === null ? undefined : JSON.stringify(body), + }); + if (allow404 && response.status === 404) return null; + if (!response.ok) throw new Error(`CNB API ${method} ${path} failed with ${response.status}: ${await response.text()}`); + return response.status === 204 ? null : response.json(); + } + + headers(json = false) { + return { + Accept: "application/vnd.cnb.api+json", + Authorization: `Bearer ${this.token}`, + ...(json ? { "Content-Type": "application/json" } : {}), + }; + } +} + +function localAssets(dir) { + return readdirSync(dir) + .map((name) => join(dir, name)) + .filter((path) => statSync(path).isFile()) + .sort((left, right) => statSync(right).size - statSync(left).size || basename(left).localeCompare(basename(right))); +} + +async function mapWithConcurrency(items, concurrency, worker) { + let nextIndex = 0; + async function runWorker() { + while (nextIndex < items.length) { + const index = nextIndex++; + await worker(items[index]); + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, runWorker)); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7def676b9..c920bd52b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -355,25 +355,34 @@ jobs: needs: publish runs-on: ubuntu-latest steps: - - name: Sync release assets to CNB + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Download GitHub release assets env: TAG_NAME: ${{ github.ref_name }} GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_USERNAME: ${{ github.repository_owner }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CNB_USERNAME: cnb - CNB_TOKEN: ${{ secrets.CNB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail + mkdir -p "$RUNNER_TEMP/github-release" "$RUNNER_TEMP/release-assets" + gh release view "$TAG_NAME" \ + --repo "$GITHUB_REPOSITORY" \ + --json tagName,name,body,targetCommitish,isPrerelease,isDraft,assets \ + > "$RUNNER_TEMP/github-release/release.json" + gh release download "$TAG_NAME" \ + --repo "$GITHUB_REPOSITORY" \ + --dir "$RUNNER_TEMP/release-assets" \ + --clobber - wget -q https://cnb.cool/znb/mpgrm/-/releases/download/v0.1.0/mpgrm_linux_amd64.tar.gz - tar -xf mpgrm_linux_amd64.tar.gz - chmod +x mpgrm - - ./mpgrm releases sync \ - --repo "https://github.com/${GITHUB_REPOSITORY}.git" \ - --target-repo "https://cnb.cool/${CNB_REPOSITORY}.git" \ - --tags "${TAG_NAME}" + - name: Sync release assets to CNB + env: + CNB_TOKEN: ${{ secrets.CNB_TOKEN }} + CNB_UPLOAD_CONCURRENCY: "3" + run: | + node .github/scripts/sync-cnb-release.mjs \ + --github-release "$RUNNER_TEMP/github-release/release.json" \ + --assets-dir "$RUNNER_TEMP/release-assets" sync-release-to-atomgit: name: Sync release to AtomGit @@ -416,6 +425,7 @@ jobs: env: ATOMGIT_TOKEN: ${{ secrets.ATOMGIT_TOKEN }} ATOMGIT_REPOSITORY: "t8y2/dbx" + ATOMGIT_UPLOAD_CONCURRENCY: "3" run: | set -euo pipefail node .github/scripts/sync-atomgit-release.mjs \