From 1670974fa435a28de3c6915c30c42b00157e2cd5 Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Thu, 25 Jun 2026 21:38:28 +0800 Subject: [PATCH] feat: add knowledge base build script --- package.json | 1 + scripts/build-knowledge-base.mjs | 278 +++++++++++++++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 scripts/build-knowledge-base.mjs diff --git a/package.json b/package.json index 150e438cb..97650f824 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "release:agents": "node scripts/release.mjs agents", "release:app": "node scripts/release.mjs app", "release:packages": "node scripts/release.mjs packages", + "build:knowledge-base": "node scripts/build-knowledge-base.mjs", "typecheck": "vue-tsc --noEmit --project apps/desktop/tsconfig.json", "build": "vite build --config apps/desktop/vite.config.ts", "build:checked": "pnpm typecheck && pnpm build", diff --git a/scripts/build-knowledge-base.mjs b/scripts/build-knowledge-base.mjs new file mode 100644 index 000000000..10aaa97b8 --- /dev/null +++ b/scripts/build-knowledge-base.mjs @@ -0,0 +1,278 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + +const DEFAULT_INPUT = "docs/content/docs"; +const DEFAULT_OUTPUT = "outputs/knowledge-base/DBX_knowledge_base_cn.md"; +const DEFAULT_LANG = "cn"; + +await main(); + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const inputDir = path.resolve(args.input ?? DEFAULT_INPUT); + const outputFile = path.resolve(args.output ?? DEFAULT_OUTPUT); + const lang = args.lang ?? DEFAULT_LANG; + const fileSuffix = lang ? `.${lang}.mdx` : ".mdx"; + + const files = (await fs.readdir(inputDir)) + .filter((name) => name.endsWith(fileSuffix)) + .sort((a, b) => a.localeCompare(b)); + + if (files.length === 0) { + throw new Error(`No ${fileSuffix} files found in ${inputDir}`); + } + + const documents = []; + let entryCount = 0; + + for (const file of files) { + const sourcePath = path.join(inputDir, file); + const displayPath = path.relative(process.cwd(), sourcePath).replaceAll(path.sep, "/"); + const raw = await fs.readFile(sourcePath, "utf8"); + const { fields, body } = parseFrontmatter(raw); + const title = fields.title || file.replace(/\.mdx$/, ""); + const description = fields.description || ""; + const sections = []; + + for (const section of splitSections(body)) { + const clean = normalizeMdx(section.lines.join("\n")); + if (!clean || clean.length < 20) continue; + + const chunks = chunkText(clean).map((chunk, index, all) => { + const heading = all.length > 1 ? `${section.heading}（${index + 1}/${all.length}）` : section.heading; + return { + heading, + question: questionFor(title, section.heading, chunk), + keywords: keywordsFor(title, section.heading, description, chunk), + answer: chunk, + }; + }); + + entryCount += chunks.length; + sections.push(...chunks); + } + + documents.push({ source: displayPath, title, description, sections }); + } + + await fs.mkdir(path.dirname(outputFile), { recursive: true }); + await fs.writeFile(outputFile, renderMarkdown(documents), "utf8"); + + console.log(`Knowledge base written: ${path.relative(process.cwd(), outputFile)}`); + console.log(`Sources: ${files.length}`); + console.log(`Entries: ${entryCount}`); +} + +function parseArgs(argv) { + const parsed = {}; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--help" || arg === "-h") { + printHelp(); + process.exit(0); + } + if (!arg.startsWith("--")) { + throw new Error(`Unexpected argument: ${arg}`); + } + const key = arg.slice(2); + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`Missing value for ${arg}`); + } + parsed[key] = value; + index += 1; + } + return parsed; +} + +function printHelp() { + console.log(`Usage: node scripts/build-knowledge-base.mjs [options] + +Options: + --input MDX docs directory. Default: ${DEFAULT_INPUT} + --output Output Markdown path. Default: ${DEFAULT_OUTPUT} + --lang File language suffix, e.g. cn for *.cn.mdx. Default: ${DEFAULT_LANG} +`); +} + +function parseFrontmatter(source) { + const match = source.match(/^---\n([\s\S]*?)\n---\n?/); + const fields = {}; + if (!match) return { fields, body: source }; + + for (const line of match[1].split("\n")) { + const separator = line.indexOf(":"); + if (separator === -1) continue; + const key = line.slice(0, separator).trim(); + const value = line.slice(separator + 1).trim().replace(/^["']|["']$/g, ""); + fields[key] = value; + } + + return { fields, body: source.slice(match[0].length) }; +} + +function splitSections(body) { + const sections = []; + let current = { heading: "概览", lines: [] }; + + for (const line of body.split("\n")) { + const heading = line.match(/^(#{2,3})\s+(.+?)\s*$/); + if (heading) { + if (current.lines.join("\n").trim()) sections.push(current); + current = { heading: heading[2].replace(/`/g, "").trim(), lines: [] }; + continue; + } + current.lines.push(line); + } + + if (current.lines.join("\n").trim()) sections.push(current); + return sections; +} + +function normalizeMdx(text) { + return text + .replace(/]*)>([\s\S]*?)<\/Card>/g, (_, attrs, inner) => { + const title = attrs.match(/title=["']([^"']+)["']/)?.[1] ?? ""; + const href = attrs.match(/href=["']([^"']+)["']/)?.[1] ?? ""; + return [title && `**${title}**`, normalizeMdx(inner), href && `参考链接：${href}`].filter(Boolean).join("\n\n"); + }) + .replace(/]*>([\s\S]*?)<\/Callout>/g, (_, inner) => normalizeMdx(inner)) + .replace(/]*>([\s\S]*?)<\/Tab>/g, (_, inner) => normalizeMdx(inner)) + .replace(/]*>([\s\S]*?)<\/Step>/g, (_, inner) => normalizeMdx(inner)) + .replace(/<[^>\n]+>/g, "") + .replace(/\{\/\*[\s\S]*?\*\/\}/g, "") + .replace(/\[([^\]]+)\]\(([^)]+)\)/g, "$1（$2）") + .replace(/^\s{0,6}#{1,6}\s+(.+)$/gm, "$1") + .replace(/^\s*import\s+.*$/gm, "") + .replace(/^\s*export\s+.*$/gm, "") + .replace(/[ \t]+\n/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +function chunkText(text, maxLength = 2200) { + if (text.length <= maxLength) return [text]; + + const chunks = []; + let current = ""; + for (const paragraph of text.split(/\n{2,}/)) { + const next = [current, paragraph].filter(Boolean).join("\n\n"); + if (next.length > maxLength && current) { + chunks.push(current); + current = paragraph; + } else { + current = next; + } + } + if (current) chunks.push(current); + + return chunks.flatMap((chunk) => { + if (chunk.length <= maxLength) return [chunk]; + const pieces = []; + for (let index = 0; index < chunk.length; index += maxLength) { + pieces.push(chunk.slice(index, index + maxLength)); + } + return pieces; + }); +} + +function questionFor(category, heading, text) { + if (heading === "概览") return `DBX 的${category}是什么？`; + if (/安装|配置|导入|导出|创建|连接|使用流程|快速开始|启动|构建|下载|打开|执行|添加|删除|更新/.test(heading)) { + return `DBX 中如何${heading.replace(/^(第一步|第二步|第三步|第四步|第五步)：/, "")}？`; + } + if (/常见|问题|故障|错误|诊断|冲突|无法/.test(heading) || /失败|报错|检查/.test(text)) { + return `DBX ${heading}时应该检查什么？`; + } + if (/支持|覆盖|矩阵|类型|数据库|文件|供应商|快捷键|命令|端口/.test(heading)) { + return `DBX ${heading}支持哪些内容？`; + } + return `DBX 的${heading}是什么？`; +} + +function keywordsFor(category, heading, description, text) { + const pool = `${category} ${heading} ${description} ${text}`; + const candidates = [ + "DBX", + "AI", + "SQL", + "MCP", + "JDBC", + "SSH", + "Docker", + "CLI", + "MongoDB", + "Redis", + "MySQL", + "PostgreSQL", + "SQLite", + "DuckDB", + "ClickHouse", + "Oracle", + "SQL Server", + "Schema", + "ER", + "导入", + "导出", + "连接", + "表", + "字段", + "查询", + "快捷键", + "插件", + "驱动", + "配置", + "安全", + "数据传输", + "格式化", + "代码片段", + "隧道", + "生产环境", + ]; + const found = candidates.filter((word) => pool.includes(word)); + return [...new Set([category, heading, ...found])].slice(0, 14).join(", "); +} + +function renderMarkdown(documents) { + const lines = [ + "# DBX 中文知识库", + "", + "> 从 `docs/content/docs/*.cn.mdx` 生成的纯 Markdown 知识库内容。", + "", + "## 来源索引", + "", + "| 来源文件 | 标题 | 描述 | 知识条数 |", + "| --- | --- | --- | ---: |", + ]; + + for (const document of documents) { + lines.push( + `| ${escapeTableCell(document.source)} | ${escapeTableCell(document.title)} | ${escapeTableCell(document.description)} | ${document.sections.length} |`, + ); + } + + for (const document of documents) { + lines.push("", `## ${document.title}`, ""); + if (document.description) lines.push(document.description, ""); + lines.push(`来源：\`${document.source}\``, ""); + + for (const section of document.sections) { + lines.push(`### ${section.heading}`, ""); + lines.push(`问题：${section.question}`, ""); + lines.push(`关键词：${section.keywords}`, ""); + lines.push("答案：", ""); + lines.push(section.answer, ""); + lines.push("---", ""); + } + } + + return `${lines.join("\n").replace(/\n{4,}/g, "\n\n\n").trim()}\n`; +} + +function escapeTableCell(value) { + return String(value ?? "") + .replaceAll("\\", "\\\\") + .replaceAll("|", "\\|") + .replace(/\s+/g, " ") + .trim(); +}