fix(agents): reuse effective release artifacts
This commit is contained in:
t8y2
parent
4858fed870
commit
12323e406c
|
|
@ -3,6 +3,8 @@ import { execFileSync } from "node:child_process";
|
|||
import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
|
||||
const VERSIONS_PATH = "agents/versions.json";
|
||||
const VERSION_SYNC_SUBJECT = "chore: bump module versions [skip ci]";
|
||||
const JRE_BUILD_PATHS = new Set([".github/workflows/agents-release.yml"]);
|
||||
|
||||
function bumpPatchVersion(version) {
|
||||
const match = /^(\d+)\.(\d+)\.(\d+)(.*)$/.exec(version);
|
||||
|
|
@ -48,6 +50,7 @@ const nativeDriverDirectories = {
|
|||
kingbase: "kingbase-go",
|
||||
rabbitmq: "rabbitmq",
|
||||
};
|
||||
const nativeDriverModules = new Set(["duckdb", "oracle", "xugu", "kingbase", "rabbitmq"]);
|
||||
|
||||
function resolveAgentModule(moduleName, { legacyStandaloneModules, moduleExists, readModuleFile }) {
|
||||
let checkDir = null;
|
||||
|
|
@ -70,10 +73,18 @@ function resolveAgentModule(moduleName, { legacyStandaloneModules, moduleExists,
|
|||
return {
|
||||
checkDir,
|
||||
modulePath,
|
||||
javaBuild: hasBuildGradle,
|
||||
nativeBuild: nativeDriverModules.has(moduleName),
|
||||
commonDependent: hasBuildGradle && (explicitlyDependsOnCommon || !legacyStandaloneModules.has(moduleName)),
|
||||
};
|
||||
}
|
||||
|
||||
function classifyModules(versions, options) {
|
||||
return Object.keys(versions)
|
||||
.map((moduleName) => ({ moduleName, module: resolveAgentModule(moduleName, options) }))
|
||||
.filter(({ module }) => module);
|
||||
}
|
||||
|
||||
export function evaluateAgentVersionBump({
|
||||
versions,
|
||||
prevVersions = versions,
|
||||
|
|
@ -87,6 +98,11 @@ export function evaluateAgentVersionBump({
|
|||
const nextVersions = { ...versions };
|
||||
const logs = [];
|
||||
let changed = false;
|
||||
const changedModules = [];
|
||||
const javaModules = [];
|
||||
const nativeModules = [];
|
||||
const reusedModules = [];
|
||||
const resolvedModules = classifyModules(versions, { legacyStandaloneModules, moduleExists, readModuleFile });
|
||||
|
||||
if (manualVersionsChanged && !skipBump) {
|
||||
logs.push("Manual agents/versions.json changes detected; preserving manually changed module versions and auto-bumping the rest.");
|
||||
|
|
@ -94,7 +110,12 @@ export function evaluateAgentVersionBump({
|
|||
|
||||
if (skipBump) {
|
||||
logs.push("Skipping automatic module version bump for migrated first release; versions.json was carried over from dbx-agents.");
|
||||
return { changed, versions: nextVersions, prevVersions, logs };
|
||||
for (const { moduleName, module } of resolvedModules) {
|
||||
changedModules.push(moduleName);
|
||||
if (module.javaBuild) javaModules.push(moduleName);
|
||||
if (module.nativeBuild) nativeModules.push(moduleName);
|
||||
}
|
||||
return { changed, versions: nextVersions, prevVersions, logs, changedModules, javaModules, nativeModules, reusedModules };
|
||||
}
|
||||
|
||||
const commonChanged = changedFiles.some(isCommonRuntimeChange);
|
||||
|
|
@ -102,10 +123,7 @@ export function evaluateAgentVersionBump({
|
|||
logs.push("Common agent runtime changes detected; common-triggered bumps are limited to modules that package agents/common.");
|
||||
}
|
||||
|
||||
for (const moduleName of Object.keys(versions)) {
|
||||
const module = resolveAgentModule(moduleName, { legacyStandaloneModules, moduleExists, readModuleFile });
|
||||
if (!module) continue;
|
||||
|
||||
for (const { moduleName, module } of resolvedModules) {
|
||||
const moduleChanged = pathChanged(changedFiles, module.modulePath);
|
||||
// Only modules that package agents/common need installer-visible updates
|
||||
// for shared Java runtime changes; native and standalone agents do not.
|
||||
|
|
@ -113,16 +131,24 @@ export function evaluateAgentVersionBump({
|
|||
const oldVersion = nextVersions[moduleName] ?? "0.1.0";
|
||||
const prevVersion = prevVersions[moduleName] ?? "";
|
||||
const manuallyVersioned = manualVersionsChanged && (!prevVersion || prevVersion !== oldVersion);
|
||||
const moduleNeedsBuild = moduleChanged || commonAffectsModule || manuallyVersioned;
|
||||
|
||||
if (!moduleChanged && !commonAffectsModule) {
|
||||
if (!moduleNeedsBuild) {
|
||||
logs.push(` ${moduleName}: no changes`);
|
||||
reusedModules.push(moduleName);
|
||||
} else if (manuallyVersioned) {
|
||||
changedModules.push(moduleName);
|
||||
if (module.javaBuild) javaModules.push(moduleName);
|
||||
if (module.nativeBuild) nativeModules.push(moduleName);
|
||||
if (!prevVersion) {
|
||||
logs.push(` ${moduleName}: CHANGED, new module version kept at ${oldVersion}`);
|
||||
} else {
|
||||
logs.push(` ${moduleName}: CHANGED, manual version ${prevVersion} -> ${oldVersion}`);
|
||||
}
|
||||
} else {
|
||||
changedModules.push(moduleName);
|
||||
if (module.javaBuild) javaModules.push(moduleName);
|
||||
if (module.nativeBuild) nativeModules.push(moduleName);
|
||||
const newVersion = bumpPatchVersion(oldVersion);
|
||||
nextVersions[moduleName] = newVersion;
|
||||
changed = true;
|
||||
|
|
@ -131,7 +157,7 @@ export function evaluateAgentVersionBump({
|
|||
}
|
||||
}
|
||||
|
||||
return { changed, versions: nextVersions, prevVersions, logs };
|
||||
return { changed, versions: nextVersions, prevVersions, logs, changedModules, javaModules, nativeModules, reusedModules };
|
||||
}
|
||||
|
||||
export function getAgentVersionChanges(previousVersions, nextVersions) {
|
||||
|
|
@ -148,6 +174,62 @@ function git(args) {
|
|||
return execFileSync("git", args, { encoding: "utf8" }).trim();
|
||||
}
|
||||
|
||||
function lines(value) {
|
||||
return value.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
export function resolveAgentReleaseBaseline({ prevTag, headRef = "HEAD", gitOutput = git }) {
|
||||
const allChangedFiles = lines(gitOutput(["diff", "--name-only", `${prevTag}..${headRef}`]));
|
||||
const versionCommits = lines(
|
||||
gitOutput([
|
||||
"log",
|
||||
"--reverse",
|
||||
"--ancestry-path",
|
||||
"--format=%H%x09%s",
|
||||
`${prevTag}..${headRef}`,
|
||||
"--",
|
||||
VERSIONS_PATH,
|
||||
]),
|
||||
);
|
||||
|
||||
let syncCommit = "";
|
||||
for (const entry of versionCommits) {
|
||||
const separator = entry.indexOf("\t");
|
||||
if (separator < 0 || entry.slice(separator + 1) !== VERSION_SYNC_SUBJECT) continue;
|
||||
|
||||
const commit = entry.slice(0, separator);
|
||||
const changedPaths = lines(gitOutput(["diff-tree", "--no-commit-id", "--name-only", "-r", commit]));
|
||||
if (changedPaths.length !== 1 || changedPaths[0] !== VERSIONS_PATH) continue;
|
||||
|
||||
JSON.parse(gitOutput(["show", `${commit}:${VERSIONS_PATH}`]));
|
||||
syncCommit = commit;
|
||||
break;
|
||||
}
|
||||
|
||||
const versionsRef = syncCommit || prevTag;
|
||||
const versions = JSON.parse(gitOutput(["show", `${versionsRef}:${VERSIONS_PATH}`]));
|
||||
const versionsChangedAfterSync = syncCommit
|
||||
? lines(gitOutput(["log", "--format=%H", `${syncCommit}..${headRef}`, "--", VERSIONS_PATH])).length > 0
|
||||
: false;
|
||||
const changedFiles = syncCommit && !versionsChangedAfterSync
|
||||
? allChangedFiles.filter((file) => file !== VERSIONS_PATH)
|
||||
: allChangedFiles;
|
||||
|
||||
return {
|
||||
prevTag,
|
||||
versionsRef,
|
||||
syncCommit,
|
||||
versions,
|
||||
changedFiles,
|
||||
allChangedFiles,
|
||||
versionsChangedAfterSync,
|
||||
};
|
||||
}
|
||||
|
||||
export function shouldBuildAgentJre(changedFiles, migratedFirstRelease = false) {
|
||||
return migratedFirstRelease || changedFiles.some((file) => JRE_BUILD_PATHS.has(file));
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
migratedFirstRelease: false,
|
||||
|
|
@ -180,7 +262,7 @@ function parseArgs(argv) {
|
|||
return options;
|
||||
}
|
||||
|
||||
function outputStepValues(result, prevTag, migratedFirstRelease) {
|
||||
function outputStepValues(result, baseline, migratedFirstRelease, buildJre) {
|
||||
const outputPath = process.env.GITHUB_OUTPUT;
|
||||
if (!outputPath) return;
|
||||
|
||||
|
|
@ -189,7 +271,14 @@ function outputStepValues(result, prevTag, migratedFirstRelease) {
|
|||
[
|
||||
`versions=${JSON.stringify(result.versions)}`,
|
||||
`prev_versions=${JSON.stringify(result.prevVersions)}`,
|
||||
`prev_tag=${prevTag}`,
|
||||
`prev_tag=${baseline.prevTag}`,
|
||||
`effective_prev_ref=${baseline.versionsRef}`,
|
||||
`changed_modules=${JSON.stringify(result.changedModules)}`,
|
||||
`java_modules=${JSON.stringify(result.javaModules)}`,
|
||||
`native_modules=${JSON.stringify(result.nativeModules)}`,
|
||||
`reuse_modules=${JSON.stringify(migratedFirstRelease ? [] : result.reusedModules)}`,
|
||||
`build_jre=${buildJre}`,
|
||||
`reuse_jre=${!migratedFirstRelease && !buildJre}`,
|
||||
`migrated_first_release=${migratedFirstRelease}`,
|
||||
"",
|
||||
].join("\n"),
|
||||
|
|
@ -200,13 +289,18 @@ function main() {
|
|||
const options = parseArgs(process.argv.slice(2));
|
||||
const versions = JSON.parse(readFileSync(VERSIONS_PATH, "utf8"));
|
||||
const legacyStandaloneModules = parseLegacyStandaloneProjects(readFileSync("agents/build.gradle", "utf8"));
|
||||
const changedFiles = options.skipBump ? [] : git(["diff", "--name-only", `${options.prevTag}..HEAD`]).split("\n").filter(Boolean);
|
||||
const baseline = options.prevVersionsFile
|
||||
? {
|
||||
prevTag: options.prevTag,
|
||||
versionsRef: options.prevTag,
|
||||
syncCommit: "",
|
||||
versions: JSON.parse(readFileSync(options.prevVersionsFile, "utf8")),
|
||||
changedFiles: lines(git(["diff", "--name-only", `${options.prevTag}..HEAD`])),
|
||||
}
|
||||
: resolveAgentReleaseBaseline({ prevTag: options.prevTag });
|
||||
const changedFiles = options.skipBump ? [] : baseline.changedFiles;
|
||||
const manualVersionsChanged = changedFiles.includes(VERSIONS_PATH);
|
||||
const prevVersions = options.prevVersionsFile
|
||||
? JSON.parse(readFileSync(options.prevVersionsFile, "utf8"))
|
||||
: manualVersionsChanged
|
||||
? JSON.parse(git(["show", `${options.prevTag}:${VERSIONS_PATH}`]))
|
||||
: versions;
|
||||
const prevVersions = baseline.versions;
|
||||
|
||||
const result = evaluateAgentVersionBump({
|
||||
versions,
|
||||
|
|
@ -226,7 +320,8 @@ function main() {
|
|||
writeFileSync(VERSIONS_PATH, versionsJson);
|
||||
}
|
||||
console.log(versionsJson);
|
||||
outputStepValues(result, options.prevTag, options.migratedFirstRelease);
|
||||
const buildJre = shouldBuildAgentJre(baseline.changedFiles, options.migratedFirstRelease);
|
||||
outputStepValues(result, baseline, options.migratedFirstRelease, buildJre);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import { evaluateAgentVersionBump } from "./bump-agent-versions.mjs";
|
||||
import { evaluateAgentVersionBump, resolveAgentReleaseBaseline, shouldBuildAgentJre } from "./bump-agent-versions.mjs";
|
||||
|
||||
const moduleExists = (path) => path === "agents/drivers/duckdb";
|
||||
|
||||
|
|
@ -40,3 +44,124 @@ test("bumps the native RabbitMQ agent from its Go directory", () => {
|
|||
|
||||
assert.equal(result.versions.rabbitmq, "0.1.1");
|
||||
});
|
||||
|
||||
test("builds a manually versioned module even without runtime file changes", () => {
|
||||
const result = evaluateAgentVersionBump({
|
||||
versions: { duckdb: "0.1.1" },
|
||||
prevVersions: { duckdb: "0.1.0" },
|
||||
changedFiles: ["agents/versions.json"],
|
||||
moduleExists,
|
||||
readModuleFile: () => "",
|
||||
});
|
||||
|
||||
assert.deepEqual(result.changedModules, ["duckdb"]);
|
||||
assert.deepEqual(result.nativeModules, ["duckdb"]);
|
||||
assert.deepEqual(result.reusedModules, []);
|
||||
assert.equal(result.versions.duckdb, "0.1.1");
|
||||
});
|
||||
|
||||
test("builds only common-dependent Java modules for a shared runtime change", () => {
|
||||
const existing = new Set([
|
||||
"agents/drivers/access",
|
||||
"agents/drivers/access/build.gradle",
|
||||
"agents/drivers/mongodb",
|
||||
"agents/drivers/mongodb/build.gradle",
|
||||
]);
|
||||
const result = evaluateAgentVersionBump({
|
||||
versions: { access: "0.1.0", mongodb: "0.1.0" },
|
||||
changedFiles: ["agents/common/src/main/java/com/dbx/Agent.java"],
|
||||
legacyStandaloneModules: new Set(["mongodb"]),
|
||||
moduleExists: (path) => existing.has(path),
|
||||
readModuleFile: () => "",
|
||||
});
|
||||
|
||||
assert.deepEqual(result.changedModules, ["access"]);
|
||||
assert.deepEqual(result.javaModules, ["access"]);
|
||||
assert.deepEqual(result.reusedModules, ["mongodb"]);
|
||||
assert.equal(result.versions.access, "0.1.1");
|
||||
assert.equal(result.versions.mongodb, "0.1.0");
|
||||
});
|
||||
|
||||
test("rebuilds JREs only for the first migration or release recipe changes", () => {
|
||||
assert.equal(shouldBuildAgentJre(["agents/drivers/access/src/main/java/Agent.java"]), false);
|
||||
assert.equal(shouldBuildAgentJre([".github/workflows/agents-release.yml"]), true);
|
||||
assert.equal(shouldBuildAgentJre([], true), true);
|
||||
});
|
||||
|
||||
test("uses the first post-tag version sync as the effective release baseline", () => {
|
||||
const repository = createRepository({ kingbase: "0.1.0" });
|
||||
git(repository, ["tag", "agents-v0.2.72"]);
|
||||
|
||||
writeVersions(repository, { kingbase: "0.1.1" });
|
||||
commitAll(repository, "chore: bump module versions [skip ci]");
|
||||
const syncCommit = git(repository, ["rev-parse", "HEAD"]);
|
||||
|
||||
writeFileSync(join(repository, "agents/drivers/kingbase-go/kingbase_metadata.go"), "package main\n\nconst fixed = true\n");
|
||||
commitAll(repository, "fix(kingbase): export primary key columns");
|
||||
|
||||
const baseline = resolveAgentReleaseBaseline({
|
||||
prevTag: "agents-v0.2.72",
|
||||
gitOutput: (args) => git(repository, args),
|
||||
});
|
||||
|
||||
assert.equal(baseline.versionsRef, syncCommit);
|
||||
assert.deepEqual(baseline.versions, { kingbase: "0.1.1" });
|
||||
assert.deepEqual(baseline.changedFiles, ["agents/drivers/kingbase-go/kingbase_metadata.go"]);
|
||||
|
||||
const result = evaluateAgentVersionBump({
|
||||
versions: { kingbase: "0.1.1" },
|
||||
prevVersions: baseline.versions,
|
||||
changedFiles: baseline.changedFiles,
|
||||
moduleExists: (path) => path === "agents/drivers/kingbase-go",
|
||||
readModuleFile: () => "",
|
||||
});
|
||||
assert.equal(result.versions.kingbase, "0.1.2");
|
||||
assert.deepEqual(result.nativeModules, ["kingbase"]);
|
||||
});
|
||||
|
||||
test("keeps versions.json publish-relevant when it changes after the sync commit", () => {
|
||||
const repository = createRepository({ duckdb: "0.1.0" });
|
||||
git(repository, ["tag", "agents-v0.2.72"]);
|
||||
|
||||
writeVersions(repository, { duckdb: "0.1.1" });
|
||||
commitAll(repository, "chore: bump module versions [skip ci]");
|
||||
writeVersions(repository, { duckdb: "0.1.2" });
|
||||
commitAll(repository, "chore: adjust DuckDB agent version");
|
||||
|
||||
const baseline = resolveAgentReleaseBaseline({
|
||||
prevTag: "agents-v0.2.72",
|
||||
gitOutput: (args) => git(repository, args),
|
||||
});
|
||||
|
||||
assert.equal(baseline.versionsChangedAfterSync, true);
|
||||
assert.deepEqual(baseline.versions, { duckdb: "0.1.1" });
|
||||
assert.deepEqual(baseline.changedFiles, ["agents/versions.json"]);
|
||||
});
|
||||
|
||||
function createRepository(versions) {
|
||||
const repository = mkdtempSync(join(tmpdir(), "dbx-agent-release-"));
|
||||
git(repository, ["init", "--initial-branch=main"]);
|
||||
git(repository, ["config", "user.name", "DBX Test"]);
|
||||
git(repository, ["config", "user.email", "dbx-test@example.com"]);
|
||||
mkdirSync(join(repository, "agents/drivers/kingbase-go"), { recursive: true });
|
||||
mkdirSync(join(repository, "agents/drivers/duckdb"), { recursive: true });
|
||||
writeVersions(repository, versions);
|
||||
writeFileSync(join(repository, "agents/drivers/kingbase-go/kingbase_metadata.go"), "package main\n");
|
||||
writeFileSync(join(repository, "agents/drivers/duckdb/Cargo.toml"), "[package]\nname = \"duckdb-test\"\n");
|
||||
commitAll(repository, "feat(agents): initial release state");
|
||||
return repository;
|
||||
}
|
||||
|
||||
function writeVersions(repository, versions) {
|
||||
mkdirSync(join(repository, "agents"), { recursive: true });
|
||||
writeFileSync(join(repository, "agents/versions.json"), `${JSON.stringify(versions, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function commitAll(repository, message) {
|
||||
git(repository, ["add", "."]);
|
||||
git(repository, ["commit", "-m", message]);
|
||||
}
|
||||
|
||||
function git(repository, args) {
|
||||
return execFileSync("git", args, { cwd: repository, encoding: "utf8" }).trim();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,371 @@
|
|||
#!/usr/bin/env node
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
copyFileSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
} from "node:fs";
|
||||
import { basename, join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
const REGISTRY_ASSET = "agent-registry.json";
|
||||
const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "rabbitmq"]);
|
||||
const PLATFORMS = [
|
||||
"macos-aarch64",
|
||||
"macos-x64",
|
||||
"linux-aarch64",
|
||||
"linux-x64",
|
||||
"windows-aarch64",
|
||||
"windows-x64",
|
||||
];
|
||||
|
||||
function artifactFilename(url) {
|
||||
return basename(url.split(/[?#]/, 1)[0]);
|
||||
}
|
||||
|
||||
function sha256(path) {
|
||||
return createHash("sha256").update(readFileSync(path)).digest("hex");
|
||||
}
|
||||
|
||||
function releaseAssetMap(release) {
|
||||
return new Map((release.assets ?? []).map((asset) => [asset.name, asset]));
|
||||
}
|
||||
|
||||
function requireReleaseAsset(assets, artifact, context) {
|
||||
const name = artifactFilename(artifact.url);
|
||||
const releaseAsset = assets.get(name);
|
||||
if (!releaseAsset) {
|
||||
throw new Error(`${context} is missing from the previous GitHub release: ${name}`);
|
||||
}
|
||||
if (!artifact.sha256) {
|
||||
throw new Error(`${context} is missing sha256 in the previous agent registry: ${name}`);
|
||||
}
|
||||
if (releaseAsset.digest !== `sha256:${artifact.sha256}`) {
|
||||
throw new Error(`${context} digest mismatch between the registry and GitHub release: ${name}`);
|
||||
}
|
||||
return { name, sha256: artifact.sha256, size: artifact.size, releaseAsset };
|
||||
}
|
||||
|
||||
export function collectReusableAssetPlan({ registry, release, versions, modules, reuseJre }) {
|
||||
const assets = releaseAssetMap(release);
|
||||
const driverAssets = [];
|
||||
const jreAssets = [];
|
||||
|
||||
for (const moduleName of modules) {
|
||||
const driver = registry.drivers?.[moduleName];
|
||||
if (!driver) {
|
||||
throw new Error(`Previous agent registry is missing reusable module: ${moduleName}`);
|
||||
}
|
||||
if (driver.version !== versions[moduleName]) {
|
||||
throw new Error(`Previous agent version mismatch for ${moduleName}: registry=${driver.version}, expected=${versions[moduleName]}`);
|
||||
}
|
||||
|
||||
if (driver.jar) {
|
||||
driverAssets.push({
|
||||
...requireReleaseAsset(assets, driver.jar, `${moduleName} Java package`),
|
||||
moduleName,
|
||||
kind: "jar",
|
||||
platform: "",
|
||||
});
|
||||
}
|
||||
|
||||
const nativePlatforms = Object.keys(driver.native ?? {}).sort();
|
||||
if (NATIVE_MODULES.has(moduleName)) {
|
||||
const missingPlatforms = PLATFORMS.filter((platform) => !nativePlatforms.includes(platform));
|
||||
const extraPlatforms = nativePlatforms.filter((platform) => !PLATFORMS.includes(platform));
|
||||
if (missingPlatforms.length > 0 || extraPlatforms.length > 0) {
|
||||
throw new Error(
|
||||
`Previous native artifacts are incomplete for ${moduleName}: missing=${missingPlatforms.join(",") || "none"}, extra=${extraPlatforms.join(",") || "none"}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const platform of nativePlatforms) {
|
||||
driverAssets.push({
|
||||
...requireReleaseAsset(assets, driver.native[platform], `${moduleName}/${platform} native package`),
|
||||
moduleName,
|
||||
kind: "native",
|
||||
platform,
|
||||
});
|
||||
}
|
||||
|
||||
if (!driver.jar && nativePlatforms.length === 0) {
|
||||
throw new Error(`Previous agent registry has no reusable artifacts for module: ${moduleName}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (reuseJre) {
|
||||
for (const [jreKey, jre] of Object.entries(registry.jres ?? {})) {
|
||||
const platforms = Object.keys(jre.platforms ?? {}).sort();
|
||||
const missingPlatforms = PLATFORMS.filter((platform) => !platforms.includes(platform));
|
||||
const extraPlatforms = platforms.filter((platform) => !PLATFORMS.includes(platform));
|
||||
if (missingPlatforms.length > 0 || extraPlatforms.length > 0) {
|
||||
throw new Error(
|
||||
`Previous JRE ${jreKey} artifacts are incomplete: missing=${missingPlatforms.join(",") || "none"}, extra=${extraPlatforms.join(",") || "none"}`,
|
||||
);
|
||||
}
|
||||
for (const platform of platforms) {
|
||||
jreAssets.push({
|
||||
...requireReleaseAsset(assets, jre.platforms[platform], `JRE ${jreKey}/${platform} package`),
|
||||
jreKey,
|
||||
platform,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (jreAssets.length === 0) {
|
||||
throw new Error("Previous agent registry has no reusable JRE artifacts.");
|
||||
}
|
||||
}
|
||||
|
||||
return { driverAssets, jreAssets };
|
||||
}
|
||||
|
||||
function verifyDownloadedAsset(path, asset) {
|
||||
if (!existsSync(path)) {
|
||||
throw new Error(`Downloaded release asset is missing: ${asset.name}`);
|
||||
}
|
||||
const size = statSync(path).size;
|
||||
if (asset.size != null && size !== asset.size) {
|
||||
throw new Error(`Downloaded release asset size mismatch for ${asset.name}: got=${size}, expected=${asset.size}`);
|
||||
}
|
||||
const digest = sha256(path);
|
||||
if (digest !== asset.sha256) {
|
||||
throw new Error(`Downloaded release asset SHA-256 mismatch for ${asset.name}: got=${digest}, expected=${asset.sha256}`);
|
||||
}
|
||||
}
|
||||
|
||||
function copyWithoutConflict(source, target) {
|
||||
if (existsSync(target)) {
|
||||
if (sha256(source) !== sha256(target)) {
|
||||
throw new Error(`Reused raw artifact conflicts with an existing file: ${basename(target)}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
copyFileSync(source, target);
|
||||
}
|
||||
|
||||
function extractRawDriver(packagePath, asset, outputDir = "") {
|
||||
const extractDir = mkdtempSync(join(tmpdir(), "dbx-agent-package-"));
|
||||
try {
|
||||
execFileSync("tar", ["--use-compress-program=unzstd", "-xf", packagePath, "-C", extractDir], { stdio: "inherit" });
|
||||
const embeddedRegistry = JSON.parse(readFileSync(join(extractDir, REGISTRY_ASSET), "utf8"));
|
||||
const driver = embeddedRegistry.drivers?.[asset.moduleName];
|
||||
if (!driver || driver.version !== asset.releaseVersion) {
|
||||
throw new Error(`Embedded registry mismatch in ${asset.name}`);
|
||||
}
|
||||
|
||||
const embeddedArtifact = asset.kind === "jar" ? driver.jar : driver.native?.[asset.platform];
|
||||
if (!embeddedArtifact) {
|
||||
throw new Error(`Embedded registry artifact is missing in ${asset.name}`);
|
||||
}
|
||||
const rawName = artifactFilename(embeddedArtifact.url);
|
||||
const rawPath = join(extractDir, "drivers", rawName);
|
||||
if (!existsSync(rawPath)) {
|
||||
throw new Error(`Embedded raw driver is missing in ${asset.name}: ${rawName}`);
|
||||
}
|
||||
const rawSize = statSync(rawPath).size;
|
||||
if (embeddedArtifact.size != null && rawSize !== embeddedArtifact.size) {
|
||||
throw new Error(`Embedded raw driver size mismatch in ${asset.name}: ${rawName}`);
|
||||
}
|
||||
if (embeddedArtifact.sha256 && sha256(rawPath) !== embeddedArtifact.sha256) {
|
||||
throw new Error(`Embedded raw driver SHA-256 mismatch in ${asset.name}: ${rawName}`);
|
||||
}
|
||||
if (outputDir) {
|
||||
copyWithoutConflict(rawPath, join(outputDir, rawName));
|
||||
}
|
||||
} finally {
|
||||
rmSync(extractDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export function extractReusableDriverPackages({ packagesDir, outputDir, versions, modules }) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
const filenames = new Set(readdirSync(packagesDir));
|
||||
let extracted = 0;
|
||||
|
||||
for (const moduleName of modules) {
|
||||
const releaseVersion = versions[moduleName];
|
||||
if (!releaseVersion) {
|
||||
throw new Error(`Missing effective previous version for reusable module: ${moduleName}`);
|
||||
}
|
||||
|
||||
const javaName = `dbx-agent-${moduleName}-${releaseVersion}.tar.zst`;
|
||||
if (filenames.has(javaName)) {
|
||||
extractRawDriver(join(packagesDir, javaName), {
|
||||
name: javaName,
|
||||
moduleName,
|
||||
kind: "jar",
|
||||
platform: "",
|
||||
releaseVersion,
|
||||
}, outputDir);
|
||||
extracted += 1;
|
||||
}
|
||||
|
||||
const nativePlatforms = [];
|
||||
for (const platform of PLATFORMS) {
|
||||
const nativeName = `dbx-agent-${moduleName}-${releaseVersion}-${platform}.tar.zst`;
|
||||
if (!filenames.has(nativeName)) continue;
|
||||
nativePlatforms.push(platform);
|
||||
extractRawDriver(join(packagesDir, nativeName), {
|
||||
name: nativeName,
|
||||
moduleName,
|
||||
kind: "native",
|
||||
platform,
|
||||
releaseVersion,
|
||||
}, outputDir);
|
||||
extracted += 1;
|
||||
}
|
||||
|
||||
if (NATIVE_MODULES.has(moduleName) && nativePlatforms.length !== PLATFORMS.length) {
|
||||
throw new Error(`Reusable native package set is incomplete for ${moduleName}.`);
|
||||
}
|
||||
if (!filenames.has(javaName) && nativePlatforms.length === 0) {
|
||||
throw new Error(`Reusable package is missing for module: ${moduleName}`);
|
||||
}
|
||||
}
|
||||
|
||||
return extracted;
|
||||
}
|
||||
|
||||
function gh(args, options = {}) {
|
||||
const result = execFileSync("gh", args, { encoding: "utf8", ...options });
|
||||
return typeof result === "string" ? result.trim() : "";
|
||||
}
|
||||
|
||||
function downloadReleaseAssets(assets, downloadDir) {
|
||||
if (assets.length === 0) return;
|
||||
const args = [
|
||||
"--fail",
|
||||
"--location",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--retry",
|
||||
"5",
|
||||
"--retry-all-errors",
|
||||
"--retry-delay",
|
||||
"2",
|
||||
"--connect-timeout",
|
||||
"30",
|
||||
"--parallel",
|
||||
"--parallel-immediate",
|
||||
"--parallel-max",
|
||||
"6",
|
||||
];
|
||||
for (const asset of assets) {
|
||||
if (!asset.browser_download_url) {
|
||||
throw new Error(`GitHub release asset is missing browser_download_url: ${asset.name}`);
|
||||
}
|
||||
args.push("--output", join(downloadDir, asset.name), asset.browser_download_url);
|
||||
}
|
||||
execFileSync("curl", args, { stdio: "inherit" });
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
repo: "",
|
||||
tag: "",
|
||||
versions: {},
|
||||
modules: [],
|
||||
reuseJre: false,
|
||||
outputDir: "",
|
||||
extractPackagesDir: "",
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
const value = argv[++index];
|
||||
if (value == null) throw new Error(`Missing value for ${arg}`);
|
||||
if (arg === "--repo") options.repo = value;
|
||||
else if (arg === "--tag") options.tag = value;
|
||||
else if (arg === "--versions") options.versions = JSON.parse(value);
|
||||
else if (arg === "--modules") options.modules = JSON.parse(value);
|
||||
else if (arg === "--reuse-jre") options.reuseJre = value === "true";
|
||||
else if (arg === "--output") options.outputDir = value;
|
||||
else if (arg === "--extract-packages") options.extractPackagesDir = value;
|
||||
else throw new Error(`Unexpected argument: ${arg}`);
|
||||
}
|
||||
|
||||
const requiredKeys = options.extractPackagesDir ? ["outputDir"] : ["repo", "tag", "outputDir"];
|
||||
for (const key of requiredKeys) {
|
||||
if (!options[key]) throw new Error(`--${key.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`)} is required.`);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.extractPackagesDir) {
|
||||
const count = extractReusableDriverPackages({
|
||||
packagesDir: options.extractPackagesDir,
|
||||
outputDir: options.outputDir,
|
||||
versions: options.versions,
|
||||
modules: options.modules,
|
||||
});
|
||||
console.log(`Extracted ${count} reusable driver artifacts.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const workDir = mkdtempSync(join(tmpdir(), "dbx-agent-reuse-"));
|
||||
const downloadDir = join(workDir, "downloads");
|
||||
mkdirSync(downloadDir);
|
||||
mkdirSync(options.outputDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const release = JSON.parse(gh(["api", `repos/${options.repo}/releases/tags/${options.tag}`]));
|
||||
const assets = releaseAssetMap(release);
|
||||
const registryReleaseAsset = assets.get(REGISTRY_ASSET);
|
||||
if (!registryReleaseAsset?.digest?.startsWith("sha256:")) {
|
||||
throw new Error(`Previous GitHub release ${options.tag} is missing a SHA-256 digest for ${REGISTRY_ASSET}.`);
|
||||
}
|
||||
|
||||
downloadReleaseAssets([registryReleaseAsset], downloadDir);
|
||||
const registryPath = join(downloadDir, REGISTRY_ASSET);
|
||||
const registryDigest = registryReleaseAsset.digest.slice("sha256:".length);
|
||||
verifyDownloadedAsset(registryPath, {
|
||||
name: REGISTRY_ASSET,
|
||||
sha256: registryDigest,
|
||||
size: registryReleaseAsset.size,
|
||||
});
|
||||
const registry = JSON.parse(readFileSync(registryPath, "utf8"));
|
||||
const plan = collectReusableAssetPlan({
|
||||
registry,
|
||||
release,
|
||||
versions: options.versions,
|
||||
modules: options.modules,
|
||||
reuseJre: options.reuseJre,
|
||||
});
|
||||
const plannedAssets = [...plan.driverAssets, ...plan.jreAssets].map((asset) => ({
|
||||
...asset,
|
||||
releaseVersion: options.versions[asset.moduleName],
|
||||
}));
|
||||
|
||||
if (plannedAssets.length > 0) {
|
||||
downloadReleaseAssets(plannedAssets.map((asset) => asset.releaseAsset), downloadDir);
|
||||
}
|
||||
|
||||
for (const asset of plannedAssets) {
|
||||
const source = join(downloadDir, asset.name);
|
||||
verifyDownloadedAsset(source, asset);
|
||||
copyFileSync(source, join(options.outputDir, asset.name));
|
||||
if (asset.moduleName) {
|
||||
extractRawDriver(source, asset);
|
||||
}
|
||||
}
|
||||
|
||||
const outputNames = readdirSync(options.outputDir).sort();
|
||||
console.log(`Reused ${plan.driverAssets.length} driver packages and ${plan.jreAssets.length} JRE packages from ${options.tag}.`);
|
||||
console.log(outputNames.join("\n"));
|
||||
} finally {
|
||||
rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main();
|
||||
}
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { collectReusableAssetPlan } from "./reuse-agent-release-assets.mjs";
|
||||
|
||||
const platforms = [
|
||||
"macos-aarch64",
|
||||
"macos-x64",
|
||||
"linux-aarch64",
|
||||
"linux-x64",
|
||||
"windows-aarch64",
|
||||
"windows-x64",
|
||||
];
|
||||
|
||||
test("collects complete reusable Java, native, and JRE assets", () => {
|
||||
const access = artifact("dbx-agent-access-0.1.34.tar.zst", "a");
|
||||
const kingbase = Object.fromEntries(
|
||||
platforms.map((platform, index) => [platform, artifact(`dbx-agent-kingbase-0.1.40-${platform}.tar.zst`, String(index + 1))]),
|
||||
);
|
||||
const jre = Object.fromEntries(
|
||||
platforms.map((platform, index) => [platform, artifact(`dbx-jre-21-${platform}.tar.zst`, String(index + 7))]),
|
||||
);
|
||||
const registry = {
|
||||
drivers: {
|
||||
access: { version: "0.1.34", jar: access },
|
||||
kingbase: { version: "0.1.40", native: kingbase },
|
||||
},
|
||||
jres: { 21: { version: "21.0.12", platforms: jre } },
|
||||
};
|
||||
const release = releaseFor([access, ...Object.values(kingbase), ...Object.values(jre)]);
|
||||
|
||||
const plan = collectReusableAssetPlan({
|
||||
registry,
|
||||
release,
|
||||
versions: { access: "0.1.34", kingbase: "0.1.40" },
|
||||
modules: ["access", "kingbase"],
|
||||
reuseJre: true,
|
||||
});
|
||||
|
||||
assert.equal(plan.driverAssets.length, 7);
|
||||
assert.equal(plan.jreAssets.length, 6);
|
||||
assert.deepEqual(plan.driverAssets.map((asset) => asset.moduleName), ["access", ...Array(6).fill("kingbase")]);
|
||||
});
|
||||
|
||||
test("rejects an incomplete reusable native platform set", () => {
|
||||
const native = Object.fromEntries(
|
||||
platforms.slice(1).map((platform, index) => [platform, artifact(`dbx-agent-duckdb-0.1.2-${platform}.tar.zst`, String(index + 1))]),
|
||||
);
|
||||
const registry = { drivers: { duckdb: { version: "0.1.2", native } }, jres: {} };
|
||||
|
||||
assert.throws(
|
||||
() => collectReusableAssetPlan({
|
||||
registry,
|
||||
release: releaseFor(Object.values(native)),
|
||||
versions: { duckdb: "0.1.2" },
|
||||
modules: ["duckdb"],
|
||||
reuseJre: false,
|
||||
}),
|
||||
/missing=macos-aarch64/,
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects a registry version that differs from the effective baseline", () => {
|
||||
const access = artifact("dbx-agent-access-0.1.33.tar.zst", "a");
|
||||
const registry = { drivers: { access: { version: "0.1.33", jar: access } }, jres: {} };
|
||||
|
||||
assert.throws(
|
||||
() => collectReusableAssetPlan({
|
||||
registry,
|
||||
release: releaseFor([access]),
|
||||
versions: { access: "0.1.34" },
|
||||
modules: ["access"],
|
||||
reuseJre: false,
|
||||
}),
|
||||
/registry=0\.1\.33, expected=0\.1\.34/,
|
||||
);
|
||||
});
|
||||
|
||||
function artifact(name, seed) {
|
||||
return { url: `https://example.invalid/${name}`, size: 100, sha256: seed.repeat(64).slice(0, 64) };
|
||||
}
|
||||
|
||||
function releaseFor(artifacts) {
|
||||
return {
|
||||
assets: artifacts.map((entry) => ({
|
||||
name: entry.url.split("/").at(-1),
|
||||
size: entry.size,
|
||||
digest: `sha256:${entry.sha256}`,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
|
@ -18,6 +18,13 @@ jobs:
|
|||
versions: ${{ steps.bump.outputs.versions }}
|
||||
prev_versions: ${{ steps.bump.outputs.prev_versions }}
|
||||
prev_tag: ${{ steps.bump.outputs.prev_tag }}
|
||||
effective_prev_ref: ${{ steps.bump.outputs.effective_prev_ref }}
|
||||
changed_modules: ${{ steps.bump.outputs.changed_modules }}
|
||||
java_modules: ${{ steps.bump.outputs.java_modules }}
|
||||
native_modules: ${{ steps.bump.outputs.native_modules }}
|
||||
reuse_modules: ${{ steps.bump.outputs.reuse_modules }}
|
||||
build_jre: ${{ steps.bump.outputs.build_jre }}
|
||||
reuse_jre: ${{ steps.bump.outputs.reuse_jre }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
|
|
@ -121,6 +128,7 @@ jobs:
|
|||
|
||||
build-agents:
|
||||
needs: [bump-versions]
|
||||
if: ${{ needs.bump-versions.outputs.java_modules != '[]' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
@ -129,7 +137,16 @@ jobs:
|
|||
distribution: temurin
|
||||
java-version: "21"
|
||||
- uses: gradle/actions/setup-gradle@v4
|
||||
- run: ./gradlew shadowJar --parallel
|
||||
- name: Build changed Java agents
|
||||
env:
|
||||
JAVA_MODULES: ${{ needs.bump-versions.outputs.java_modules }}
|
||||
run: |
|
||||
mapfile -t MODULES < <(echo "$JAVA_MODULES" | python3 -c 'import json,sys; print("\n".join(json.load(sys.stdin)))')
|
||||
TASKS=()
|
||||
for module in "${MODULES[@]}"; do
|
||||
TASKS+=(":${module}:shadowJar")
|
||||
done
|
||||
./gradlew "${TASKS[@]}" --parallel
|
||||
working-directory: agents
|
||||
- run: python3 scripts/validate_agent_jars.py
|
||||
working-directory: agents
|
||||
|
|
@ -141,6 +158,7 @@ jobs:
|
|||
|
||||
build-oracle-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'oracle') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
@ -180,6 +198,7 @@ jobs:
|
|||
|
||||
build-xugu-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'xugu') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
@ -219,6 +238,7 @@ jobs:
|
|||
|
||||
build-rabbitmq-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'rabbitmq') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
@ -258,6 +278,7 @@ jobs:
|
|||
|
||||
build-kingbase-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'kingbase') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
@ -297,6 +318,7 @@ jobs:
|
|||
|
||||
build-duckdb-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'duckdb') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
|
@ -401,6 +423,8 @@ jobs:
|
|||
path: "release-native/dbx-agent-duckdb-*"
|
||||
|
||||
build-jre:
|
||||
needs: [bump-versions]
|
||||
if: ${{ needs.bump-versions.outputs.build_jre == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
|
|
@ -474,8 +498,41 @@ jobs:
|
|||
name: jre-${{ matrix.jre-key }}
|
||||
path: "dbx-jre-*.tar.zst"
|
||||
|
||||
reuse-previous-assets:
|
||||
name: Reuse unchanged agent artifacts
|
||||
needs: [bump-versions]
|
||||
if: ${{ needs.bump-versions.outputs.reuse_modules != '[]' || needs.bump-versions.outputs.reuse_jre == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install artifact tools
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y zstd
|
||||
- name: Download and verify previous release artifacts
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PREV_TAG: ${{ needs.bump-versions.outputs.prev_tag }}
|
||||
PREV_VERSIONS: ${{ needs.bump-versions.outputs.prev_versions }}
|
||||
REUSE_MODULES: ${{ needs.bump-versions.outputs.reuse_modules }}
|
||||
REUSE_JRE: ${{ needs.bump-versions.outputs.reuse_jre }}
|
||||
run: |
|
||||
node .github/scripts/reuse-agent-release-assets.mjs \
|
||||
--repo "$GITHUB_REPOSITORY" \
|
||||
--tag "$PREV_TAG" \
|
||||
--versions "$PREV_VERSIONS" \
|
||||
--modules "$REUSE_MODULES" \
|
||||
--reuse-jre "$REUSE_JRE" \
|
||||
--output reused-release
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: reused-agent-artifacts
|
||||
path: reused-release/*
|
||||
retention-days: 1
|
||||
|
||||
release:
|
||||
needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-kingbase-native, build-duckdb-native, build-jre]
|
||||
needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-kingbase-native, build-duckdb-native, build-jre, reuse-previous-assets]
|
||||
if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create DBX bot release token
|
||||
|
|
@ -499,18 +556,32 @@ jobs:
|
|||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: Install artifact tools
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y zstd
|
||||
|
||||
- name: Flatten artifacts
|
||||
run: |
|
||||
mkdir -p release
|
||||
find artifacts/agent-jars -name '*.jar' -exec cp {} release/ \;
|
||||
find artifacts/oracle-native -type f -name 'dbx-agent-oracle-*' -exec cp {} release/ \;
|
||||
find artifacts/xugu-native -type f -name 'dbx-agent-xugu-*' -exec cp {} release/ \;
|
||||
find artifacts/rabbitmq-native -type f -name 'dbx-agent-rabbitmq-*' -exec cp {} release/ \;
|
||||
find artifacts/kingbase-native -type f -name 'dbx-agent-kingbase-*' -exec cp {} release/ \;
|
||||
find artifacts/duckdb-native-* -type f -name 'dbx-agent-duckdb-*' -exec cp {} release/ \;
|
||||
find artifacts -type f -name 'dbx-agent-*.jar' -exec cp {} release/ \;
|
||||
find artifacts -type f -name 'dbx-agent-*' ! -name '*.jar' ! -name '*.tar.zst' -exec cp {} release/ \;
|
||||
find artifacts -type f -name 'dbx-agent-*.tar.zst' -exec cp {} release/ \;
|
||||
find artifacts -name 'dbx-jre-*.tar.zst' -exec cp {} release/ \;
|
||||
ls -lh release/
|
||||
|
||||
- name: Extract reused raw agent artifacts
|
||||
if: ${{ needs.bump-versions.outputs.reuse_modules != '[]' }}
|
||||
env:
|
||||
PREV_VERSIONS: ${{ needs.bump-versions.outputs.prev_versions }}
|
||||
REUSE_MODULES: ${{ needs.bump-versions.outputs.reuse_modules }}
|
||||
run: |
|
||||
node .github/scripts/reuse-agent-release-assets.mjs \
|
||||
--extract-packages release \
|
||||
--versions "$PREV_VERSIONS" \
|
||||
--modules "$REUSE_MODULES" \
|
||||
--output release
|
||||
|
||||
- name: Add versions to agent artifact filenames
|
||||
env:
|
||||
MODULE_VERSIONS: ${{ needs.bump-versions.outputs.versions }}
|
||||
|
|
@ -681,8 +752,6 @@ jobs:
|
|||
|
||||
- name: Build single-driver packages
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y zstd
|
||||
python3 agents/scripts/build_driver_zips.py release --cleanup-sources
|
||||
python3 -m json.tool release/agent-registry.json > /dev/null
|
||||
echo "=== final agent-registry.json ==="
|
||||
|
|
|
|||
|
|
@ -166,9 +166,9 @@ git push origin "$RELEASE_TAG"
|
|||
|
||||
The release workflow will:
|
||||
|
||||
- Bump changed module versions in `versions.json`.
|
||||
- Build all agent shadow jars.
|
||||
- Build/download JRE `.tar.zst` artifacts.
|
||||
- Resolve the effective previous module versions from the post-release version-sync commit after the previous `agents-v*` tag.
|
||||
- Bump and build only changed Java or native agent modules.
|
||||
- Download unchanged single-driver packages and JRE archives from the previous immutable release, then verify filenames, versions, platform coverage, sizes, and SHA-256 digests before reuse.
|
||||
- Generate `agent-registry.json`.
|
||||
- Create full offline platform ZIPs from raw staging files.
|
||||
- Create one `.tar.zst` package per Java or native driver.
|
||||
|
|
|
|||
|
|
@ -81,7 +81,10 @@ def build_driver_zips(release_dir: Path) -> list[Path]:
|
|||
package_driver["jar"] = packaged_artifact(jar_artifact, source)
|
||||
package_registry = {"jres": {}, "drivers": {driver_name: package_driver}}
|
||||
output = release_dir / f"dbx-agent-{driver_name}-{version}.tar.zst"
|
||||
write_driver_tar_zstd(output, package_registry, source, executable=False)
|
||||
if not output.exists():
|
||||
write_driver_tar_zstd(output, package_registry, source, executable=False)
|
||||
elif not output.is_file():
|
||||
raise FileExistsError(f"Reusable Java agent package is not a file: {output}")
|
||||
update_release_artifact(jar_artifact, output)
|
||||
outputs.append(output)
|
||||
|
||||
|
|
@ -96,7 +99,10 @@ def build_driver_zips(release_dir: Path) -> list[Path]:
|
|||
package_driver["native"] = {platform: packaged_artifact(artifact, source)}
|
||||
package_registry = {"jres": {}, "drivers": {driver_name: package_driver}}
|
||||
output = release_dir / f"dbx-agent-{driver_name}-{version}-{platform}.tar.zst"
|
||||
write_driver_tar_zstd(output, package_registry, source, executable=True)
|
||||
if not output.exists():
|
||||
write_driver_tar_zstd(output, package_registry, source, executable=True)
|
||||
elif not output.is_file():
|
||||
raise FileExistsError(f"Reusable native agent package is not a file: {output}")
|
||||
update_release_artifact(artifact, output)
|
||||
outputs.append(output)
|
||||
|
||||
|
|
@ -121,7 +127,7 @@ def main() -> None:
|
|||
args = parser.parse_args()
|
||||
|
||||
for path in build_driver_zips(args.release_dir):
|
||||
print(f"Created {path.name} ({path.stat().st_size} bytes)")
|
||||
print(f"Prepared {path.name} ({path.stat().st_size} bytes)")
|
||||
if args.cleanup_sources:
|
||||
for path in remove_raw_driver_artifacts(args.release_dir):
|
||||
print(f"Removed intermediate {path.name}")
|
||||
|
|
|
|||
|
|
@ -340,7 +340,7 @@ agents/drivers/<驱动模块名>/build/libs/
|
|||
#### `agents/versions.json` 什么时候修改
|
||||
|
||||
<Callout type="warn">
|
||||
修改已有驱动时，不要手动修改 `agents/versions.json`。Agent 发布工作流会对比上一个 `agents-v*` 标签，自动为发生变化的模块增加 patch 版本。
|
||||
修改已有驱动时，不要手动修改 `agents/versions.json`。Agent 发布工作流会以上一个 `agents-v*` 标签对比运行时代码，并使用该版本发布后的版本同步提交作为有效版本基线，自动为发生变化的模块增加 patch 版本；未变化模块会复用上一不可变 Release 中经过校验的产物。
|
||||
</Callout>
|
||||
|
||||
- 修改 `agents/drivers/<module>/`：发布时自动 bump 该模块版本
|
||||
|
|
|
|||
|
|
@ -340,7 +340,7 @@ Restart DBX or disconnect and reconnect the database so the old agent process ex
|
|||
#### When to Change `agents/versions.json`
|
||||
|
||||
<Callout type="warn">
|
||||
Do not edit `agents/versions.json` when changing an existing driver. The agent release workflow compares against the previous `agents-v*` tag and automatically increments the patch version of changed modules.
|
||||
Do not edit `agents/versions.json` when changing an existing driver. The agent release workflow compares runtime files against the previous `agents-v*` tag, uses that release's post-release version-sync commit as the effective version baseline, and automatically increments the patch version of changed modules. Unchanged modules reuse verified artifacts from the previous immutable release.
|
||||
</Callout>
|
||||
|
||||
- A change under `agents/drivers/<module>/` automatically bumps that module during release
|
||||
|
|
|
|||
|
|
@ -2,7 +2,13 @@ import { spawnSync } from "node:child_process";
|
|||
import { readFileSync } from "node:fs";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
import { stdin as input, stdout as output } from "node:process";
|
||||
import { evaluateAgentVersionBump, getAgentVersionChanges, isAgentPublishRelevantFile, parseLegacyStandaloneProjects } from "../.github/scripts/bump-agent-versions.mjs";
|
||||
import {
|
||||
evaluateAgentVersionBump,
|
||||
getAgentVersionChanges,
|
||||
isAgentPublishRelevantFile,
|
||||
parseLegacyStandaloneProjects,
|
||||
resolveAgentReleaseBaseline,
|
||||
} from "../.github/scripts/bump-agent-versions.mjs";
|
||||
|
||||
const REPO = "t8y2/dbx";
|
||||
const PACKAGES_WORKFLOW = "mcp-release.yml";
|
||||
|
|
@ -47,6 +53,7 @@ const AGENT_RELEASE_PATHS = [
|
|||
"agents/common/build.gradle",
|
||||
"agents/common/src/main/",
|
||||
"agents/drivers/",
|
||||
".github/workflows/agents-release.yml",
|
||||
];
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
|
|
@ -180,7 +187,7 @@ async function releaseAgents(bump) {
|
|||
console.log(kv("Release target", "Agents", bold));
|
||||
console.log(kv("Current agent tag", `${latest.tag}${latest.source ? ` (${latest.source})` : ""}`, yellow));
|
||||
printReleaseStatus(status);
|
||||
printAgentVersionChanges(latest.tag, status.changedFiles);
|
||||
printAgentVersionChanges(status);
|
||||
if (!status.needed && !force) {
|
||||
console.log(yellow("No agents release needed; publish-relevant agent runtime files have not changed."));
|
||||
console.log(dim("Use --force to create the tag anyway."));
|
||||
|
|
@ -453,11 +460,18 @@ function getAgentReleaseStatus(latest = getLatestAgentTag()) {
|
|||
};
|
||||
}
|
||||
|
||||
const changedFiles = getChangedFilesSince(latest.tag, AGENT_RELEASE_PATHS).filter(isAgentPublishRelevantFile);
|
||||
const effectiveBaseline = resolveAgentReleaseBaseline({ prevTag: latest.tag });
|
||||
const changedFiles = effectiveBaseline.changedFiles
|
||||
.filter((file) => AGENT_RELEASE_PATHS.some((path) => file === path || file.startsWith(path)))
|
||||
.filter(isAgentPublishRelevantFile);
|
||||
return {
|
||||
needed: changedFiles.length > 0,
|
||||
baseline: latest.tag,
|
||||
changedFiles,
|
||||
previousVersions: effectiveBaseline.versions,
|
||||
versionBaseline: effectiveBaseline.syncCommit
|
||||
? `${effectiveBaseline.syncCommit.slice(0, 10)} (post-release version sync)`
|
||||
: latest.tag,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -478,6 +492,9 @@ function getChangedFilesSince(ref, paths) {
|
|||
function printReleaseStatus(status) {
|
||||
console.log(kv("Release needed", status.needed ? "yes" : "no", status.needed ? yellow : green));
|
||||
console.log(kv("Compared against", status.baseline, dim));
|
||||
if (status.versionBaseline && status.versionBaseline !== status.baseline) {
|
||||
console.log(kv("Version baseline", status.versionBaseline, dim));
|
||||
}
|
||||
if (status.reason) {
|
||||
console.log(kv("Reason", status.reason, dim));
|
||||
}
|
||||
|
|
@ -492,20 +509,19 @@ function printReleaseStatus(status) {
|
|||
}
|
||||
}
|
||||
|
||||
function printAgentVersionChanges(baselineTag, changedFiles) {
|
||||
if (changedFiles.length === 0 || !refExists(`refs/tags/${baselineTag}`)) return;
|
||||
function printAgentVersionChanges(status) {
|
||||
if (status.changedFiles.length === 0 || !status.previousVersions) return;
|
||||
|
||||
const currentVersions = JSON.parse(readFileSync("agents/versions.json", "utf8"));
|
||||
const previousVersions = JSON.parse(run("git", ["show", `${baselineTag}:agents/versions.json`]).stdout);
|
||||
const legacyStandaloneModules = parseLegacyStandaloneProjects(readFileSync("agents/build.gradle", "utf8"));
|
||||
const result = evaluateAgentVersionBump({
|
||||
versions: currentVersions,
|
||||
prevVersions: previousVersions,
|
||||
changedFiles,
|
||||
prevVersions: status.previousVersions,
|
||||
changedFiles: status.changedFiles,
|
||||
legacyStandaloneModules,
|
||||
manualVersionsChanged: changedFiles.includes("agents/versions.json"),
|
||||
manualVersionsChanged: status.changedFiles.includes("agents/versions.json"),
|
||||
});
|
||||
const changes = getAgentVersionChanges(previousVersions, result.versions)
|
||||
const changes = getAgentVersionChanges(status.previousVersions, result.versions)
|
||||
.map(({ moduleName, previousVersion, nextVersion }) => `${moduleName}: ${previousVersion ?? "new"} -> ${nextVersion}`);
|
||||
|
||||
if (changes.length === 0) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue