diff --git a/.github/scripts/bump-agent-versions.mjs b/.github/scripts/bump-agent-versions.mjs index 48676b909..bc2705e8e 100644 --- a/.github/scripts/bump-agent-versions.mjs +++ b/.github/scripts/bump-agent-versions.mjs @@ -3,6 +3,8 @@ import { execFileSync } from "node:child_process"; import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs"; const VERSIONS_PATH = "agents/versions.json"; +const VERSION_SYNC_SUBJECT = "chore: bump module versions [skip ci]"; +const JRE_BUILD_PATHS = new Set([".github/workflows/agents-release.yml"]); function bumpPatchVersion(version) { const match = /^(\d+)\.(\d+)\.(\d+)(.*)$/.exec(version); @@ -48,6 +50,7 @@ const nativeDriverDirectories = { kingbase: "kingbase-go", rabbitmq: "rabbitmq", }; +const nativeDriverModules = new Set(["duckdb", "oracle", "xugu", "kingbase", "rabbitmq"]); function resolveAgentModule(moduleName, { legacyStandaloneModules, moduleExists, readModuleFile }) { let checkDir = null; @@ -70,10 +73,18 @@ function resolveAgentModule(moduleName, { legacyStandaloneModules, moduleExists, return { checkDir, modulePath, + javaBuild: hasBuildGradle, + nativeBuild: nativeDriverModules.has(moduleName), commonDependent: hasBuildGradle && (explicitlyDependsOnCommon || !legacyStandaloneModules.has(moduleName)), }; } +function classifyModules(versions, options) { + return Object.keys(versions) + .map((moduleName) => ({ moduleName, module: resolveAgentModule(moduleName, options) })) + .filter(({ module }) => module); +} + export function evaluateAgentVersionBump({ versions, prevVersions = versions, @@ -87,6 +98,11 @@ export function evaluateAgentVersionBump({ const nextVersions = { ...versions }; const logs = []; let changed = false; + const changedModules = []; + const javaModules = []; + const nativeModules = []; + const reusedModules = []; + const resolvedModules = classifyModules(versions, { legacyStandaloneModules, moduleExists, readModuleFile }); if (manualVersionsChanged && !skipBump) { logs.push("Manual agents/versions.json changes detected; preserving manually changed module versions and auto-bumping the rest."); @@ -94,7 +110,12 @@ export function evaluateAgentVersionBump({ if (skipBump) { logs.push("Skipping automatic module version bump for migrated first release; versions.json was carried over from dbx-agents."); - return { changed, versions: nextVersions, prevVersions, logs }; + for (const { moduleName, module } of resolvedModules) { + changedModules.push(moduleName); + if (module.javaBuild) javaModules.push(moduleName); + if (module.nativeBuild) nativeModules.push(moduleName); + } + return { changed, versions: nextVersions, prevVersions, logs, changedModules, javaModules, nativeModules, reusedModules }; } const commonChanged = changedFiles.some(isCommonRuntimeChange); @@ -102,10 +123,7 @@ export function evaluateAgentVersionBump({ logs.push("Common agent runtime changes detected; common-triggered bumps are limited to modules that package agents/common."); } - for (const moduleName of Object.keys(versions)) { - const module = resolveAgentModule(moduleName, { legacyStandaloneModules, moduleExists, readModuleFile }); - if (!module) continue; - + for (const { moduleName, module } of resolvedModules) { const moduleChanged = pathChanged(changedFiles, module.modulePath); // Only modules that package agents/common need installer-visible updates // for shared Java runtime changes; native and standalone agents do not. @@ -113,16 +131,24 @@ export function evaluateAgentVersionBump({ const oldVersion = nextVersions[moduleName] ?? "0.1.0"; const prevVersion = prevVersions[moduleName] ?? ""; const manuallyVersioned = manualVersionsChanged && (!prevVersion || prevVersion !== oldVersion); + const moduleNeedsBuild = moduleChanged || commonAffectsModule || manuallyVersioned; - if (!moduleChanged && !commonAffectsModule) { + if (!moduleNeedsBuild) { logs.push(` ${moduleName}: no changes`); + reusedModules.push(moduleName); } else if (manuallyVersioned) { + changedModules.push(moduleName); + if (module.javaBuild) javaModules.push(moduleName); + if (module.nativeBuild) nativeModules.push(moduleName); if (!prevVersion) { logs.push(` ${moduleName}: CHANGED, new module version kept at ${oldVersion}`); } else { logs.push(` ${moduleName}: CHANGED, manual version ${prevVersion} -> ${oldVersion}`); } } else { + changedModules.push(moduleName); + if (module.javaBuild) javaModules.push(moduleName); + if (module.nativeBuild) nativeModules.push(moduleName); const newVersion = bumpPatchVersion(oldVersion); nextVersions[moduleName] = newVersion; changed = true; @@ -131,7 +157,7 @@ export function evaluateAgentVersionBump({ } } - return { changed, versions: nextVersions, prevVersions, logs }; + return { changed, versions: nextVersions, prevVersions, logs, changedModules, javaModules, nativeModules, reusedModules }; } export function getAgentVersionChanges(previousVersions, nextVersions) { @@ -148,6 +174,62 @@ function git(args) { return execFileSync("git", args, { encoding: "utf8" }).trim(); } +function lines(value) { + return value.split(/\r?\n/).map((line) => line.trim()).filter(Boolean); +} + +export function resolveAgentReleaseBaseline({ prevTag, headRef = "HEAD", gitOutput = git }) { + const allChangedFiles = lines(gitOutput(["diff", "--name-only", `${prevTag}..${headRef}`])); + const versionCommits = lines( + gitOutput([ + "log", + "--reverse", + "--ancestry-path", + "--format=%H%x09%s", + `${prevTag}..${headRef}`, + "--", + VERSIONS_PATH, + ]), + ); + + let syncCommit = ""; + for (const entry of versionCommits) { + const separator = entry.indexOf("\t"); + if (separator < 0 || entry.slice(separator + 1) !== VERSION_SYNC_SUBJECT) continue; + + const commit = entry.slice(0, separator); + const changedPaths = lines(gitOutput(["diff-tree", "--no-commit-id", "--name-only", "-r", commit])); + if (changedPaths.length !== 1 || changedPaths[0] !== VERSIONS_PATH) continue; + + JSON.parse(gitOutput(["show", `${commit}:${VERSIONS_PATH}`])); + syncCommit = commit; + break; + } + + const versionsRef = syncCommit || prevTag; + const versions = JSON.parse(gitOutput(["show", `${versionsRef}:${VERSIONS_PATH}`])); + const versionsChangedAfterSync = syncCommit + ? lines(gitOutput(["log", "--format=%H", `${syncCommit}..${headRef}`, "--", VERSIONS_PATH])).length > 0 + : false; + const changedFiles = syncCommit && !versionsChangedAfterSync + ? allChangedFiles.filter((file) => file !== VERSIONS_PATH) + : allChangedFiles; + + return { + prevTag, + versionsRef, + syncCommit, + versions, + changedFiles, + allChangedFiles, + versionsChangedAfterSync, + }; +} + +export function shouldBuildAgentJre(changedFiles, migratedFirstRelease = false) { + return migratedFirstRelease || changedFiles.some((file) => JRE_BUILD_PATHS.has(file)); +} + function parseArgs(argv) { const options = { migratedFirstRelease: false, @@ -180,7 +262,7 @@ function parseArgs(argv) { return options; } -function outputStepValues(result, prevTag, migratedFirstRelease) { +function outputStepValues(result, baseline, migratedFirstRelease, buildJre) { const outputPath = process.env.GITHUB_OUTPUT; if (!outputPath) return; @@ -189,7 +271,14 @@ function outputStepValues(result, prevTag, migratedFirstRelease) { [ `versions=${JSON.stringify(result.versions)}`, `prev_versions=${JSON.stringify(result.prevVersions)}`, - `prev_tag=${prevTag}`, + `prev_tag=${baseline.prevTag}`, + `effective_prev_ref=${baseline.versionsRef}`, + `changed_modules=${JSON.stringify(result.changedModules)}`, + `java_modules=${JSON.stringify(result.javaModules)}`, + `native_modules=${JSON.stringify(result.nativeModules)}`, + `reuse_modules=${JSON.stringify(migratedFirstRelease ? [] : result.reusedModules)}`, + `build_jre=${buildJre}`, + `reuse_jre=${!migratedFirstRelease && !buildJre}`, `migrated_first_release=${migratedFirstRelease}`, "", ].join("\n"), @@ -200,13 +289,18 @@ function main() { const options = parseArgs(process.argv.slice(2)); const versions = JSON.parse(readFileSync(VERSIONS_PATH, "utf8")); const legacyStandaloneModules = parseLegacyStandaloneProjects(readFileSync("agents/build.gradle", "utf8")); - const changedFiles = options.skipBump ? [] : git(["diff", "--name-only", `${options.prevTag}..HEAD`]).split("\n").filter(Boolean); + const baseline = options.prevVersionsFile + ? { + prevTag: options.prevTag, + versionsRef: options.prevTag, + syncCommit: "", + versions: JSON.parse(readFileSync(options.prevVersionsFile, "utf8")), + changedFiles: lines(git(["diff", "--name-only", `${options.prevTag}..HEAD`])), + } + : resolveAgentReleaseBaseline({ prevTag: options.prevTag }); + const changedFiles = options.skipBump ? [] : baseline.changedFiles; const manualVersionsChanged = changedFiles.includes(VERSIONS_PATH); - const prevVersions = options.prevVersionsFile - ? JSON.parse(readFileSync(options.prevVersionsFile, "utf8")) - : manualVersionsChanged - ? JSON.parse(git(["show", `${options.prevTag}:${VERSIONS_PATH}`])) - : versions; + const prevVersions = baseline.versions; const result = evaluateAgentVersionBump({ versions, @@ -226,7 +320,8 @@ function main() { writeFileSync(VERSIONS_PATH, versionsJson); } console.log(versionsJson); - outputStepValues(result, options.prevTag, options.migratedFirstRelease); + const buildJre = shouldBuildAgentJre(baseline.changedFiles, options.migratedFirstRelease); + outputStepValues(result, baseline, options.migratedFirstRelease, buildJre); } if (import.meta.url === `file://${process.argv[1]}`) { diff --git a/.github/scripts/bump-agent-versions.test.mjs b/.github/scripts/bump-agent-versions.test.mjs index c3bd0f94a..e2c1b5e30 100644 --- a/.github/scripts/bump-agent-versions.test.mjs +++ b/.github/scripts/bump-agent-versions.test.mjs @@ -1,7 +1,11 @@ import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import test from "node:test"; -import { evaluateAgentVersionBump } from "./bump-agent-versions.mjs"; +import { evaluateAgentVersionBump, resolveAgentReleaseBaseline, shouldBuildAgentJre } from "./bump-agent-versions.mjs"; const moduleExists = (path) => path === "agents/drivers/duckdb"; @@ -40,3 +44,124 @@ test("bumps the native RabbitMQ agent from its Go directory", () => { assert.equal(result.versions.rabbitmq, "0.1.1"); }); + +test("builds a manually versioned module even without runtime file changes", () => { + const result = evaluateAgentVersionBump({ + versions: { duckdb: "0.1.1" }, + prevVersions: { duckdb: "0.1.0" }, + changedFiles: ["agents/versions.json"], + moduleExists, + readModuleFile: () => "", + }); + + assert.deepEqual(result.changedModules, ["duckdb"]); + assert.deepEqual(result.nativeModules, ["duckdb"]); + assert.deepEqual(result.reusedModules, []); + assert.equal(result.versions.duckdb, "0.1.1"); +}); + +test("builds only common-dependent Java modules for a shared runtime change", () => { + const existing = new Set([ + "agents/drivers/access", + "agents/drivers/access/build.gradle", + "agents/drivers/mongodb", + "agents/drivers/mongodb/build.gradle", + ]); + const result = evaluateAgentVersionBump({ + versions: { access: "0.1.0", mongodb: "0.1.0" }, + changedFiles: ["agents/common/src/main/java/com/dbx/Agent.java"], + legacyStandaloneModules: new Set(["mongodb"]), + moduleExists: (path) => existing.has(path), + readModuleFile: () => "", + }); + + assert.deepEqual(result.changedModules, ["access"]); + assert.deepEqual(result.javaModules, ["access"]); + assert.deepEqual(result.reusedModules, ["mongodb"]); + assert.equal(result.versions.access, "0.1.1"); + assert.equal(result.versions.mongodb, "0.1.0"); +}); + +test("rebuilds JREs only for the first migration or release recipe changes", () => { + assert.equal(shouldBuildAgentJre(["agents/drivers/access/src/main/java/Agent.java"]), false); + assert.equal(shouldBuildAgentJre([".github/workflows/agents-release.yml"]), true); + assert.equal(shouldBuildAgentJre([], true), true); +}); + +test("uses the first post-tag version sync as the effective release baseline", () => { + const repository = createRepository({ kingbase: "0.1.0" }); + git(repository, ["tag", "agents-v0.2.72"]); + + writeVersions(repository, { kingbase: "0.1.1" }); + commitAll(repository, "chore: bump module versions [skip ci]"); + const syncCommit = git(repository, ["rev-parse", "HEAD"]); + + writeFileSync(join(repository, "agents/drivers/kingbase-go/kingbase_metadata.go"), "package main\n\nconst fixed = true\n"); + commitAll(repository, "fix(kingbase): export primary key columns"); + + const baseline = resolveAgentReleaseBaseline({ + prevTag: "agents-v0.2.72", + gitOutput: (args) => git(repository, args), + }); + + assert.equal(baseline.versionsRef, syncCommit); + assert.deepEqual(baseline.versions, { kingbase: "0.1.1" }); + assert.deepEqual(baseline.changedFiles, ["agents/drivers/kingbase-go/kingbase_metadata.go"]); + + const result = evaluateAgentVersionBump({ + versions: { kingbase: "0.1.1" }, + prevVersions: baseline.versions, + changedFiles: baseline.changedFiles, + moduleExists: (path) => path === "agents/drivers/kingbase-go", + readModuleFile: () => "", + }); + assert.equal(result.versions.kingbase, "0.1.2"); + assert.deepEqual(result.nativeModules, ["kingbase"]); +}); + +test("keeps versions.json publish-relevant when it changes after the sync commit", () => { + const repository = createRepository({ duckdb: "0.1.0" }); + git(repository, ["tag", "agents-v0.2.72"]); + + writeVersions(repository, { duckdb: "0.1.1" }); + commitAll(repository, "chore: bump module versions [skip ci]"); + writeVersions(repository, { duckdb: "0.1.2" }); + commitAll(repository, "chore: adjust DuckDB agent version"); + + const baseline = resolveAgentReleaseBaseline({ + prevTag: "agents-v0.2.72", + gitOutput: (args) => git(repository, args), + }); + + assert.equal(baseline.versionsChangedAfterSync, true); + assert.deepEqual(baseline.versions, { duckdb: "0.1.1" }); + assert.deepEqual(baseline.changedFiles, ["agents/versions.json"]); +}); + +function createRepository(versions) { + const repository = mkdtempSync(join(tmpdir(), "dbx-agent-release-")); + git(repository, ["init", "--initial-branch=main"]); + git(repository, ["config", "user.name", "DBX Test"]); + git(repository, ["config", "user.email", "dbx-test@example.com"]); + mkdirSync(join(repository, "agents/drivers/kingbase-go"), { recursive: true }); + mkdirSync(join(repository, "agents/drivers/duckdb"), { recursive: true }); + writeVersions(repository, versions); + writeFileSync(join(repository, "agents/drivers/kingbase-go/kingbase_metadata.go"), "package main\n"); + writeFileSync(join(repository, "agents/drivers/duckdb/Cargo.toml"), "[package]\nname = \"duckdb-test\"\n"); + commitAll(repository, "feat(agents): initial release state"); + return repository; +} + +function writeVersions(repository, versions) { + mkdirSync(join(repository, "agents"), { recursive: true }); + writeFileSync(join(repository, "agents/versions.json"), `${JSON.stringify(versions, null, 2)}\n`); +} + +function commitAll(repository, message) { + git(repository, ["add", "."]); + git(repository, ["commit", "-m", message]); +} + +function git(repository, args) { + return execFileSync("git", args, { cwd: repository, encoding: "utf8" }).trim(); +} diff --git a/.github/scripts/reuse-agent-release-assets.mjs b/.github/scripts/reuse-agent-release-assets.mjs new file mode 100644 index 000000000..fb69cbb42 --- /dev/null +++ b/.github/scripts/reuse-agent-release-assets.mjs @@ -0,0 +1,371 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, +} from "node:fs"; +import { basename, join } from "node:path"; +import { tmpdir } from "node:os"; + +const REGISTRY_ASSET = "agent-registry.json"; +const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "rabbitmq"]); +const PLATFORMS = [ + "macos-aarch64", + "macos-x64", + "linux-aarch64", + "linux-x64", + "windows-aarch64", + "windows-x64", +]; + +function artifactFilename(url) { + return basename(url.split(/[?#]/, 1)[0]); +} + +function sha256(path) { + return createHash("sha256").update(readFileSync(path)).digest("hex"); +} + +function releaseAssetMap(release) { + return new Map((release.assets ?? []).map((asset) => [asset.name, asset])); +} + +function requireReleaseAsset(assets, artifact, context) { + const name = artifactFilename(artifact.url); + const releaseAsset = assets.get(name); + if (!releaseAsset) { + throw new Error(`${context} is missing from the previous GitHub release: ${name}`); + } + if (!artifact.sha256) { + throw new Error(`${context} is missing sha256 in the previous agent registry: ${name}`); + } + if (releaseAsset.digest !== `sha256:${artifact.sha256}`) { + throw new Error(`${context} digest mismatch between the registry and GitHub release: ${name}`); + } + return { name, sha256: artifact.sha256, size: artifact.size, releaseAsset }; +} + +export function collectReusableAssetPlan({ registry, release, versions, modules, reuseJre }) { + const assets = releaseAssetMap(release); + const driverAssets = []; + const jreAssets = []; + + for (const moduleName of modules) { + const driver = registry.drivers?.[moduleName]; + if (!driver) { + throw new Error(`Previous agent registry is missing reusable module: ${moduleName}`); + } + if (driver.version !== versions[moduleName]) { + throw new Error(`Previous agent version mismatch for ${moduleName}: registry=${driver.version}, expected=${versions[moduleName]}`); + } + + if (driver.jar) { + driverAssets.push({ + ...requireReleaseAsset(assets, driver.jar, `${moduleName} Java package`), + moduleName, + kind: "jar", + platform: "", + }); + } + + const nativePlatforms = Object.keys(driver.native ?? {}).sort(); + if (NATIVE_MODULES.has(moduleName)) { + const missingPlatforms = PLATFORMS.filter((platform) => !nativePlatforms.includes(platform)); + const extraPlatforms = nativePlatforms.filter((platform) => !PLATFORMS.includes(platform)); + if (missingPlatforms.length > 0 || extraPlatforms.length > 0) { + throw new Error( + `Previous native artifacts are incomplete for ${moduleName}: missing=${missingPlatforms.join(",") || "none"}, extra=${extraPlatforms.join(",") || "none"}`, + ); + } + } + + for (const platform of nativePlatforms) { + driverAssets.push({ + ...requireReleaseAsset(assets, driver.native[platform], `${moduleName}/${platform} native package`), + moduleName, + kind: "native", + platform, + }); + } + + if (!driver.jar && nativePlatforms.length === 0) { + throw new Error(`Previous agent registry has no reusable artifacts for module: ${moduleName}`); + } + } + + if (reuseJre) { + for (const [jreKey, jre] of Object.entries(registry.jres ?? {})) { + const platforms = Object.keys(jre.platforms ?? {}).sort(); + const missingPlatforms = PLATFORMS.filter((platform) => !platforms.includes(platform)); + const extraPlatforms = platforms.filter((platform) => !PLATFORMS.includes(platform)); + if (missingPlatforms.length > 0 || extraPlatforms.length > 0) { + throw new Error( + `Previous JRE ${jreKey} artifacts are incomplete: missing=${missingPlatforms.join(",") || "none"}, extra=${extraPlatforms.join(",") || "none"}`, + ); + } + for (const platform of platforms) { + jreAssets.push({ + ...requireReleaseAsset(assets, jre.platforms[platform], `JRE ${jreKey}/${platform} package`), + jreKey, + platform, + }); + } + } + if (jreAssets.length === 0) { + throw new Error("Previous agent registry has no reusable JRE artifacts."); + } + } + + return { driverAssets, jreAssets }; +} + +function verifyDownloadedAsset(path, asset) { + if (!existsSync(path)) { + throw new Error(`Downloaded release asset is missing: ${asset.name}`); + } + const size = statSync(path).size; + if (asset.size != null && size !== asset.size) { + throw new Error(`Downloaded release asset size mismatch for ${asset.name}: got=${size}, expected=${asset.size}`); + } + const digest = sha256(path); + if (digest !== asset.sha256) { + throw new Error(`Downloaded release asset SHA-256 mismatch for ${asset.name}: got=${digest}, expected=${asset.sha256}`); + } +} + +function copyWithoutConflict(source, target) { + if (existsSync(target)) { + if (sha256(source) !== sha256(target)) { + throw new Error(`Reused raw artifact conflicts with an existing file: ${basename(target)}`); + } + return; + } + copyFileSync(source, target); +} + +function extractRawDriver(packagePath, asset, outputDir = "") { + const extractDir = mkdtempSync(join(tmpdir(), "dbx-agent-package-")); + try { + execFileSync("tar", ["--use-compress-program=unzstd", "-xf", packagePath, "-C", extractDir], { stdio: "inherit" }); + const embeddedRegistry = JSON.parse(readFileSync(join(extractDir, REGISTRY_ASSET), "utf8")); + const driver = embeddedRegistry.drivers?.[asset.moduleName]; + if (!driver || driver.version !== asset.releaseVersion) { + throw new Error(`Embedded registry mismatch in ${asset.name}`); + } + + const embeddedArtifact = asset.kind === "jar" ? driver.jar : driver.native?.[asset.platform]; + if (!embeddedArtifact) { + throw new Error(`Embedded registry artifact is missing in ${asset.name}`); + } + const rawName = artifactFilename(embeddedArtifact.url); + const rawPath = join(extractDir, "drivers", rawName); + if (!existsSync(rawPath)) { + throw new Error(`Embedded raw driver is missing in ${asset.name}: ${rawName}`); + } + const rawSize = statSync(rawPath).size; + if (embeddedArtifact.size != null && rawSize !== embeddedArtifact.size) { + throw new Error(`Embedded raw driver size mismatch in ${asset.name}: ${rawName}`); + } + if (embeddedArtifact.sha256 && sha256(rawPath) !== embeddedArtifact.sha256) { + throw new Error(`Embedded raw driver SHA-256 mismatch in ${asset.name}: ${rawName}`); + } + if (outputDir) { + copyWithoutConflict(rawPath, join(outputDir, rawName)); + } + } finally { + rmSync(extractDir, { recursive: true, force: true }); + } +} + +export function extractReusableDriverPackages({ packagesDir, outputDir, versions, modules }) { + mkdirSync(outputDir, { recursive: true }); + const filenames = new Set(readdirSync(packagesDir)); + let extracted = 0; + + for (const moduleName of modules) { + const releaseVersion = versions[moduleName]; + if (!releaseVersion) { + throw new Error(`Missing effective previous version for reusable module: ${moduleName}`); + } + + const javaName = `dbx-agent-${moduleName}-${releaseVersion}.tar.zst`; + if (filenames.has(javaName)) { + extractRawDriver(join(packagesDir, javaName), { + name: javaName, + moduleName, + kind: "jar", + platform: "", + releaseVersion, + }, outputDir); + extracted += 1; + } + + const nativePlatforms = []; + for (const platform of PLATFORMS) { + const nativeName = `dbx-agent-${moduleName}-${releaseVersion}-${platform}.tar.zst`; + if (!filenames.has(nativeName)) continue; + nativePlatforms.push(platform); + extractRawDriver(join(packagesDir, nativeName), { + name: nativeName, + moduleName, + kind: "native", + platform, + releaseVersion, + }, outputDir); + extracted += 1; + } + + if (NATIVE_MODULES.has(moduleName) && nativePlatforms.length !== PLATFORMS.length) { + throw new Error(`Reusable native package set is incomplete for ${moduleName}.`); + } + if (!filenames.has(javaName) && nativePlatforms.length === 0) { + throw new Error(`Reusable package is missing for module: ${moduleName}`); + } + } + + return extracted; +} + +function gh(args, options = {}) { + const result = execFileSync("gh", args, { encoding: "utf8", ...options }); + return typeof result === "string" ? result.trim() : ""; +} + +function downloadReleaseAssets(assets, downloadDir) { + if (assets.length === 0) return; + const args = [ + "--fail", + "--location", + "--silent", + "--show-error", + "--retry", + "5", + "--retry-all-errors", + "--retry-delay", + "2", + "--connect-timeout", + "30", + "--parallel", + "--parallel-immediate", + "--parallel-max", + "6", + ]; + for (const asset of assets) { + if (!asset.browser_download_url) { + throw new Error(`GitHub release asset is missing browser_download_url: ${asset.name}`); + } + args.push("--output", join(downloadDir, asset.name), asset.browser_download_url); + } + execFileSync("curl", args, { stdio: "inherit" }); +} + +function parseArgs(argv) { + const options = { + repo: "", + tag: "", + versions: {}, + modules: [], + reuseJre: false, + outputDir: "", + extractPackagesDir: "", + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const value = argv[++index]; + if (value == null) throw new Error(`Missing value for ${arg}`); + if (arg === "--repo") options.repo = value; + else if (arg === "--tag") options.tag = value; + else if (arg === "--versions") options.versions = JSON.parse(value); + else if (arg === "--modules") options.modules = JSON.parse(value); + else if (arg === "--reuse-jre") options.reuseJre = value === "true"; + else if (arg === "--output") options.outputDir = value; + else if (arg === "--extract-packages") options.extractPackagesDir = value; + else throw new Error(`Unexpected argument: ${arg}`); + } + + const requiredKeys = options.extractPackagesDir ? ["outputDir"] : ["repo", "tag", "outputDir"]; + for (const key of requiredKeys) { + if (!options[key]) throw new Error(`--${key.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`)} is required.`); + } + return options; +} + +function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.extractPackagesDir) { + const count = extractReusableDriverPackages({ + packagesDir: options.extractPackagesDir, + outputDir: options.outputDir, + versions: options.versions, + modules: options.modules, + }); + console.log(`Extracted ${count} reusable driver artifacts.`); + return; + } + + const workDir = mkdtempSync(join(tmpdir(), "dbx-agent-reuse-")); + const downloadDir = join(workDir, "downloads"); + mkdirSync(downloadDir); + mkdirSync(options.outputDir, { recursive: true }); + + try { + const release = JSON.parse(gh(["api", `repos/${options.repo}/releases/tags/${options.tag}`])); + const assets = releaseAssetMap(release); + const registryReleaseAsset = assets.get(REGISTRY_ASSET); + if (!registryReleaseAsset?.digest?.startsWith("sha256:")) { + throw new Error(`Previous GitHub release ${options.tag} is missing a SHA-256 digest for ${REGISTRY_ASSET}.`); + } + + downloadReleaseAssets([registryReleaseAsset], downloadDir); + const registryPath = join(downloadDir, REGISTRY_ASSET); + const registryDigest = registryReleaseAsset.digest.slice("sha256:".length); + verifyDownloadedAsset(registryPath, { + name: REGISTRY_ASSET, + sha256: registryDigest, + size: registryReleaseAsset.size, + }); + const registry = JSON.parse(readFileSync(registryPath, "utf8")); + const plan = collectReusableAssetPlan({ + registry, + release, + versions: options.versions, + modules: options.modules, + reuseJre: options.reuseJre, + }); + const plannedAssets = [...plan.driverAssets, ...plan.jreAssets].map((asset) => ({ + ...asset, + releaseVersion: options.versions[asset.moduleName], + })); + + if (plannedAssets.length > 0) { + downloadReleaseAssets(plannedAssets.map((asset) => asset.releaseAsset), downloadDir); + } + + for (const asset of plannedAssets) { + const source = join(downloadDir, asset.name); + verifyDownloadedAsset(source, asset); + copyFileSync(source, join(options.outputDir, asset.name)); + if (asset.moduleName) { + extractRawDriver(source, asset); + } + } + + const outputNames = readdirSync(options.outputDir).sort(); + console.log(`Reused ${plan.driverAssets.length} driver packages and ${plan.jreAssets.length} JRE packages from ${options.tag}.`); + console.log(outputNames.join("\n")); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/.github/scripts/reuse-agent-release-assets.test.mjs b/.github/scripts/reuse-agent-release-assets.test.mjs new file mode 100644 index 000000000..69b86099a --- /dev/null +++ b/.github/scripts/reuse-agent-release-assets.test.mjs @@ -0,0 +1,91 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { collectReusableAssetPlan } from "./reuse-agent-release-assets.mjs"; + +const platforms = [ + "macos-aarch64", + "macos-x64", + "linux-aarch64", + "linux-x64", + "windows-aarch64", + "windows-x64", +]; + +test("collects complete reusable Java, native, and JRE assets", () => { + const access = artifact("dbx-agent-access-0.1.34.tar.zst", "a"); + const kingbase = Object.fromEntries( + platforms.map((platform, index) => [platform, artifact(`dbx-agent-kingbase-0.1.40-${platform}.tar.zst`, String(index + 1))]), + ); + const jre = Object.fromEntries( + platforms.map((platform, index) => [platform, artifact(`dbx-jre-21-${platform}.tar.zst`, String(index + 7))]), + ); + const registry = { + drivers: { + access: { version: "0.1.34", jar: access }, + kingbase: { version: "0.1.40", native: kingbase }, + }, + jres: { 21: { version: "21.0.12", platforms: jre } }, + }; + const release = releaseFor([access, ...Object.values(kingbase), ...Object.values(jre)]); + + const plan = collectReusableAssetPlan({ + registry, + release, + versions: { access: "0.1.34", kingbase: "0.1.40" }, + modules: ["access", "kingbase"], + reuseJre: true, + }); + + assert.equal(plan.driverAssets.length, 7); + assert.equal(plan.jreAssets.length, 6); + assert.deepEqual(plan.driverAssets.map((asset) => asset.moduleName), ["access", ...Array(6).fill("kingbase")]); +}); + +test("rejects an incomplete reusable native platform set", () => { + const native = Object.fromEntries( + platforms.slice(1).map((platform, index) => [platform, artifact(`dbx-agent-duckdb-0.1.2-${platform}.tar.zst`, String(index + 1))]), + ); + const registry = { drivers: { duckdb: { version: "0.1.2", native } }, jres: {} }; + + assert.throws( + () => collectReusableAssetPlan({ + registry, + release: releaseFor(Object.values(native)), + versions: { duckdb: "0.1.2" }, + modules: ["duckdb"], + reuseJre: false, + }), + /missing=macos-aarch64/, + ); +}); + +test("rejects a registry version that differs from the effective baseline", () => { + const access = artifact("dbx-agent-access-0.1.33.tar.zst", "a"); + const registry = { drivers: { access: { version: "0.1.33", jar: access } }, jres: {} }; + + assert.throws( + () => collectReusableAssetPlan({ + registry, + release: releaseFor([access]), + versions: { access: "0.1.34" }, + modules: ["access"], + reuseJre: false, + }), + /registry=0\.1\.33, expected=0\.1\.34/, + ); +}); + +function artifact(name, seed) { + return { url: `https://example.invalid/${name}`, size: 100, sha256: seed.repeat(64).slice(0, 64) }; +} + +function releaseFor(artifacts) { + return { + assets: artifacts.map((entry) => ({ + name: entry.url.split("/").at(-1), + size: entry.size, + digest: `sha256:${entry.sha256}`, + })), + }; +} diff --git a/.github/workflows/agents-release.yml b/.github/workflows/agents-release.yml index 45aef7246..35afd884b 100644 --- a/.github/workflows/agents-release.yml +++ b/.github/workflows/agents-release.yml @@ -18,6 +18,13 @@ jobs: versions: ${{ steps.bump.outputs.versions }} prev_versions: ${{ steps.bump.outputs.prev_versions }} prev_tag: ${{ steps.bump.outputs.prev_tag }} + effective_prev_ref: ${{ steps.bump.outputs.effective_prev_ref }} + changed_modules: ${{ steps.bump.outputs.changed_modules }} + java_modules: ${{ steps.bump.outputs.java_modules }} + native_modules: ${{ steps.bump.outputs.native_modules }} + reuse_modules: ${{ steps.bump.outputs.reuse_modules }} + build_jre: ${{ steps.bump.outputs.build_jre }} + reuse_jre: ${{ steps.bump.outputs.reuse_jre }} steps: - uses: actions/checkout@v4 with: @@ -121,6 +128,7 @@ jobs: build-agents: needs: [bump-versions] + if: ${{ needs.bump-versions.outputs.java_modules != '[]' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -129,7 +137,16 @@ jobs: distribution: temurin java-version: "21" - uses: gradle/actions/setup-gradle@v4 - - run: ./gradlew shadowJar --parallel + - name: Build changed Java agents + env: + JAVA_MODULES: ${{ needs.bump-versions.outputs.java_modules }} + run: | + mapfile -t MODULES < <(echo "$JAVA_MODULES" | python3 -c 'import json,sys; print("\n".join(json.load(sys.stdin)))') + TASKS=() + for module in "${MODULES[@]}"; do + TASKS+=(":${module}:shadowJar") + done + ./gradlew "${TASKS[@]}" --parallel working-directory: agents - run: python3 scripts/validate_agent_jars.py working-directory: agents @@ -141,6 +158,7 @@ jobs: build-oracle-native: needs: [bump-versions] + if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'oracle') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -180,6 +198,7 @@ jobs: build-xugu-native: needs: [bump-versions] + if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'xugu') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -219,6 +238,7 @@ jobs: build-rabbitmq-native: needs: [bump-versions] + if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'rabbitmq') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -258,6 +278,7 @@ jobs: build-kingbase-native: needs: [bump-versions] + if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'kingbase') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -297,6 +318,7 @@ jobs: build-duckdb-native: needs: [bump-versions] + if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'duckdb') }} strategy: fail-fast: false matrix: @@ -401,6 +423,8 @@ jobs: path: "release-native/dbx-agent-duckdb-*" build-jre: + needs: [bump-versions] + if: ${{ needs.bump-versions.outputs.build_jre == 'true' }} runs-on: ubuntu-latest strategy: matrix: @@ -474,8 +498,41 @@ jobs: name: jre-${{ matrix.jre-key }} path: "dbx-jre-*.tar.zst" + reuse-previous-assets: + name: Reuse unchanged agent artifacts + needs: [bump-versions] + if: ${{ needs.bump-versions.outputs.reuse_modules != '[]' || needs.bump-versions.outputs.reuse_jre == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install artifact tools + run: | + sudo apt-get update + sudo apt-get install -y zstd + - name: Download and verify previous release artifacts + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PREV_TAG: ${{ needs.bump-versions.outputs.prev_tag }} + PREV_VERSIONS: ${{ needs.bump-versions.outputs.prev_versions }} + REUSE_MODULES: ${{ needs.bump-versions.outputs.reuse_modules }} + REUSE_JRE: ${{ needs.bump-versions.outputs.reuse_jre }} + run: | + node .github/scripts/reuse-agent-release-assets.mjs \ + --repo "$GITHUB_REPOSITORY" \ + --tag "$PREV_TAG" \ + --versions "$PREV_VERSIONS" \ + --modules "$REUSE_MODULES" \ + --reuse-jre "$REUSE_JRE" \ + --output reused-release + - uses: actions/upload-artifact@v4 + with: + name: reused-agent-artifacts + path: reused-release/* + retention-days: 1 + release: - needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-kingbase-native, build-duckdb-native, build-jre] + needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-kingbase-native, build-duckdb-native, build-jre, reuse-previous-assets] + if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} runs-on: ubuntu-latest steps: - name: Create DBX bot release token @@ -499,18 +556,32 @@ jobs: with: path: artifacts + - name: Install artifact tools + run: | + sudo apt-get update + sudo apt-get install -y zstd + - name: Flatten artifacts run: | mkdir -p release - find artifacts/agent-jars -name '*.jar' -exec cp {} release/ \; - find artifacts/oracle-native -type f -name 'dbx-agent-oracle-*' -exec cp {} release/ \; - find artifacts/xugu-native -type f -name 'dbx-agent-xugu-*' -exec cp {} release/ \; - find artifacts/rabbitmq-native -type f -name 'dbx-agent-rabbitmq-*' -exec cp {} release/ \; - find artifacts/kingbase-native -type f -name 'dbx-agent-kingbase-*' -exec cp {} release/ \; - find artifacts/duckdb-native-* -type f -name 'dbx-agent-duckdb-*' -exec cp {} release/ \; + find artifacts -type f -name 'dbx-agent-*.jar' -exec cp {} release/ \; + find artifacts -type f -name 'dbx-agent-*' ! -name '*.jar' ! -name '*.tar.zst' -exec cp {} release/ \; + find artifacts -type f -name 'dbx-agent-*.tar.zst' -exec cp {} release/ \; find artifacts -name 'dbx-jre-*.tar.zst' -exec cp {} release/ \; ls -lh release/ + - name: Extract reused raw agent artifacts + if: ${{ needs.bump-versions.outputs.reuse_modules != '[]' }} + env: + PREV_VERSIONS: ${{ needs.bump-versions.outputs.prev_versions }} + REUSE_MODULES: ${{ needs.bump-versions.outputs.reuse_modules }} + run: | + node .github/scripts/reuse-agent-release-assets.mjs \ + --extract-packages release \ + --versions "$PREV_VERSIONS" \ + --modules "$REUSE_MODULES" \ + --output release + - name: Add versions to agent artifact filenames env: MODULE_VERSIONS: ${{ needs.bump-versions.outputs.versions }} @@ -681,8 +752,6 @@ jobs: - name: Build single-driver packages run: | - sudo apt-get update - sudo apt-get install -y zstd python3 agents/scripts/build_driver_zips.py release --cleanup-sources python3 -m json.tool release/agent-registry.json > /dev/null echo "=== final agent-registry.json ===" diff --git a/agents/docs/release-checklist.md b/agents/docs/release-checklist.md index 2b36ba4e9..2c8297992 100644 --- a/agents/docs/release-checklist.md +++ b/agents/docs/release-checklist.md @@ -166,9 +166,9 @@ git push origin "$RELEASE_TAG" The release workflow will: -- Bump changed module versions in `versions.json`. -- Build all agent shadow jars. -- Build/download JRE `.tar.zst` artifacts. +- Resolve the effective previous module versions from the post-release version-sync commit after the previous `agents-v*` tag. +- Bump and build only changed Java or native agent modules. +- Download unchanged single-driver packages and JRE archives from the previous immutable release, then verify filenames, versions, platform coverage, sizes, and SHA-256 digests before reuse. - Generate `agent-registry.json`. - Create full offline platform ZIPs from raw staging files. - Create one `.tar.zst` package per Java or native driver. diff --git a/agents/scripts/build_driver_zips.py b/agents/scripts/build_driver_zips.py index 8c6ff9f3e..a152b8824 100644 --- a/agents/scripts/build_driver_zips.py +++ b/agents/scripts/build_driver_zips.py @@ -81,7 +81,10 @@ def build_driver_zips(release_dir: Path) -> list[Path]: package_driver["jar"] = packaged_artifact(jar_artifact, source) package_registry = {"jres": {}, "drivers": {driver_name: package_driver}} output = release_dir / f"dbx-agent-{driver_name}-{version}.tar.zst" - write_driver_tar_zstd(output, package_registry, source, executable=False) + if not output.exists(): + write_driver_tar_zstd(output, package_registry, source, executable=False) + elif not output.is_file(): + raise FileExistsError(f"Reusable Java agent package is not a file: {output}") update_release_artifact(jar_artifact, output) outputs.append(output) @@ -96,7 +99,10 @@ def build_driver_zips(release_dir: Path) -> list[Path]: package_driver["native"] = {platform: packaged_artifact(artifact, source)} package_registry = {"jres": {}, "drivers": {driver_name: package_driver}} output = release_dir / f"dbx-agent-{driver_name}-{version}-{platform}.tar.zst" - write_driver_tar_zstd(output, package_registry, source, executable=True) + if not output.exists(): + write_driver_tar_zstd(output, package_registry, source, executable=True) + elif not output.is_file(): + raise FileExistsError(f"Reusable native agent package is not a file: {output}") update_release_artifact(artifact, output) outputs.append(output) @@ -121,7 +127,7 @@ def main() -> None: args = parser.parse_args() for path in build_driver_zips(args.release_dir): - print(f"Created {path.name} ({path.stat().st_size} bytes)") + print(f"Prepared {path.name} ({path.stat().st_size} bytes)") if args.cleanup_sources: for path in remove_raw_driver_artifacts(args.release_dir): print(f"Removed intermediate {path.name}") diff --git a/docs/content/docs/contributing.cn.mdx b/docs/content/docs/contributing.cn.mdx index 9f67487b5..dede27b90 100644 --- a/docs/content/docs/contributing.cn.mdx +++ b/docs/content/docs/contributing.cn.mdx @@ -340,7 +340,7 @@ agents/drivers/<驱动模块名>/build/libs/ #### `agents/versions.json` 什么时候修改 - 修改已有驱动时,不要手动修改 `agents/versions.json`。Agent 发布工作流会对比上一个 `agents-v*` 标签,自动为发生变化的模块增加 patch 版本。 + 修改已有驱动时,不要手动修改 `agents/versions.json`。Agent 发布工作流会以上一个 `agents-v*` 标签对比运行时代码,并使用该版本发布后的版本同步提交作为有效版本基线,自动为发生变化的模块增加 patch 版本;未变化模块会复用上一不可变 Release 中经过校验的产物。 - 修改 `agents/drivers//`:发布时自动 bump 该模块版本 diff --git a/docs/content/docs/contributing.mdx b/docs/content/docs/contributing.mdx index 382c70877..31eb0b992 100644 --- a/docs/content/docs/contributing.mdx +++ b/docs/content/docs/contributing.mdx @@ -340,7 +340,7 @@ Restart DBX or disconnect and reconnect the database so the old agent process ex #### When to Change `agents/versions.json` - Do not edit `agents/versions.json` when changing an existing driver. The agent release workflow compares against the previous `agents-v*` tag and automatically increments the patch version of changed modules. + Do not edit `agents/versions.json` when changing an existing driver. The agent release workflow compares runtime files against the previous `agents-v*` tag, uses that release's post-release version-sync commit as the effective version baseline, and automatically increments the patch version of changed modules. Unchanged modules reuse verified artifacts from the previous immutable release. - A change under `agents/drivers//` automatically bumps that module during release diff --git a/scripts/release.mjs b/scripts/release.mjs index 5a21d4a53..15ab3ed48 100644 --- a/scripts/release.mjs +++ b/scripts/release.mjs @@ -2,7 +2,13 @@ import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { createInterface } from "node:readline/promises"; import { stdin as input, stdout as output } from "node:process"; -import { evaluateAgentVersionBump, getAgentVersionChanges, isAgentPublishRelevantFile, parseLegacyStandaloneProjects } from "../.github/scripts/bump-agent-versions.mjs"; +import { + evaluateAgentVersionBump, + getAgentVersionChanges, + isAgentPublishRelevantFile, + parseLegacyStandaloneProjects, + resolveAgentReleaseBaseline, +} from "../.github/scripts/bump-agent-versions.mjs"; const REPO = "t8y2/dbx"; const PACKAGES_WORKFLOW = "mcp-release.yml"; @@ -47,6 +53,7 @@ const AGENT_RELEASE_PATHS = [ "agents/common/build.gradle", "agents/common/src/main/", "agents/drivers/", + ".github/workflows/agents-release.yml", ]; const args = process.argv.slice(2); @@ -180,7 +187,7 @@ async function releaseAgents(bump) { console.log(kv("Release target", "Agents", bold)); console.log(kv("Current agent tag", `${latest.tag}${latest.source ? ` (${latest.source})` : ""}`, yellow)); printReleaseStatus(status); - printAgentVersionChanges(latest.tag, status.changedFiles); + printAgentVersionChanges(status); if (!status.needed && !force) { console.log(yellow("No agents release needed; publish-relevant agent runtime files have not changed.")); console.log(dim("Use --force to create the tag anyway.")); @@ -453,11 +460,18 @@ function getAgentReleaseStatus(latest = getLatestAgentTag()) { }; } - const changedFiles = getChangedFilesSince(latest.tag, AGENT_RELEASE_PATHS).filter(isAgentPublishRelevantFile); + const effectiveBaseline = resolveAgentReleaseBaseline({ prevTag: latest.tag }); + const changedFiles = effectiveBaseline.changedFiles + .filter((file) => AGENT_RELEASE_PATHS.some((path) => file === path || file.startsWith(path))) + .filter(isAgentPublishRelevantFile); return { needed: changedFiles.length > 0, baseline: latest.tag, changedFiles, + previousVersions: effectiveBaseline.versions, + versionBaseline: effectiveBaseline.syncCommit + ? `${effectiveBaseline.syncCommit.slice(0, 10)} (post-release version sync)` + : latest.tag, }; } @@ -478,6 +492,9 @@ function getChangedFilesSince(ref, paths) { function printReleaseStatus(status) { console.log(kv("Release needed", status.needed ? "yes" : "no", status.needed ? yellow : green)); console.log(kv("Compared against", status.baseline, dim)); + if (status.versionBaseline && status.versionBaseline !== status.baseline) { + console.log(kv("Version baseline", status.versionBaseline, dim)); + } if (status.reason) { console.log(kv("Reason", status.reason, dim)); } @@ -492,20 +509,19 @@ function printReleaseStatus(status) { } } -function printAgentVersionChanges(baselineTag, changedFiles) { - if (changedFiles.length === 0 || !refExists(`refs/tags/${baselineTag}`)) return; +function printAgentVersionChanges(status) { + if (status.changedFiles.length === 0 || !status.previousVersions) return; const currentVersions = JSON.parse(readFileSync("agents/versions.json", "utf8")); - const previousVersions = JSON.parse(run("git", ["show", `${baselineTag}:agents/versions.json`]).stdout); const legacyStandaloneModules = parseLegacyStandaloneProjects(readFileSync("agents/build.gradle", "utf8")); const result = evaluateAgentVersionBump({ versions: currentVersions, - prevVersions: previousVersions, - changedFiles, + prevVersions: status.previousVersions, + changedFiles: status.changedFiles, legacyStandaloneModules, - manualVersionsChanged: changedFiles.includes("agents/versions.json"), + manualVersionsChanged: status.changedFiles.includes("agents/versions.json"), }); - const changes = getAgentVersionChanges(previousVersions, result.versions) + const changes = getAgentVersionChanges(status.previousVersions, result.versions) .map(({ moduleName, previousVersion, nextVersion }) => `${moduleName}: ${previousVersion ?? "new"} -> ${nextVersion}`); if (changes.length === 0) {