From ce0037d70a7585447844bb22d78cc31c9e8893c8 Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Sun, 17 May 2026 16:25:17 +0800 Subject: [PATCH] fix(export): page table data exports --- src/components/sidebar/TreeItem.vue | 27 +++++++++------- src/lib/tableDataExport.ts | 50 +++++++++++++++++++++++++++++ tests/tableDataExport.test.ts | 40 +++++++++++++++++++++++ 3 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 src/lib/tableDataExport.ts create mode 100644 tests/tableDataExport.test.ts diff --git a/src/components/sidebar/TreeItem.vue b/src/components/sidebar/TreeItem.vue index d02a1ae13..9355e387f 100644 --- a/src/components/sidebar/TreeItem.vue +++ b/src/components/sidebar/TreeItem.vue @@ -88,6 +88,7 @@ import { } from "@/lib/databaseCapabilities"; import { sidebarSelectionCopyAction, treeNodeRowAction, treeNodeRowDoubleClickAction } from "@/lib/treeNodeClick"; import { formatCsv, formatJson, formatSqlInsert } from "@/lib/exportFormats"; +import { fetchTableDataForExport } from "@/lib/tableDataExport"; import { buildCreateDatabaseSql, supportsCreateDatabaseCharset } from "@/lib/createDatabaseSql"; import { buildRenameObjectSql, supportsObjectRename, type RenameableObjectType } from "@/lib/objectRenameSql"; import { hexToRgba } from "@/lib/color"; @@ -1061,29 +1062,30 @@ async function exportStructure() { async function exportData(format: "csv" | "json" | "sql") { const node = props.node; if (!node.connectionId || !node.database) return; + const connectionId = node.connectionId; + const database = node.database; const config = connectionStore.getConfig(node.connectionId); if (!config) return; try { - await connectionStore.ensureConnected(node.connectionId); + await connectionStore.ensureConnected(connectionId); const qualifiedName = isSchemaAware(config.db_type) && node.schema ? `${quoteIdent(node.schema)}.${quoteIdent(node.label)}` : quoteIdent(node.label); const queryColumns = config.db_type === "neo4j" - ? (await api.getColumns(node.connectionId, node.database, node.schema || node.database, node.label)).map( + ? (await api.getColumns(connectionId, database, node.schema || database, node.label)).map( (column) => column.name, ) : undefined; - const dataSql = buildTableSelectSql({ + const result = await fetchTableDataForExport({ databaseType: config.db_type, schema: node.schema, tableName: node.label, columns: queryColumns, - limit: 10000, + executePage: (sql) => api.executeQuery(connectionId, database, sql), }); - const result = await api.executeQuery(node.connectionId, node.database, dataSql); let content: string; let ext: string; @@ -1100,7 +1102,7 @@ async function exportData(format: "csv" | "json" | "sql") { } await saveFileContent(content, `${node.label}.${ext}`, ext.toUpperCase(), ext); - toast(result.truncated ? t("grid.exported") + " (truncated)" : t("grid.exported")); + toast(t("grid.exported")); } catch (e: any) { toast(t("grid.exportFailed", { message: e?.message || String(e) }), 5000); } @@ -1109,25 +1111,26 @@ async function exportData(format: "csv" | "json" | "sql") { async function exportDataXlsx() { const node = props.node; if (!node.connectionId || !node.database) return; + const connectionId = node.connectionId; + const database = node.database; const config = connectionStore.getConfig(node.connectionId); if (!config) return; try { - await connectionStore.ensureConnected(node.connectionId); + await connectionStore.ensureConnected(connectionId); const queryColumns = config.db_type === "neo4j" - ? (await api.getColumns(node.connectionId, node.database, node.schema || node.database, node.label)).map( + ? (await api.getColumns(connectionId, database, node.schema || database, node.label)).map( (column) => column.name, ) : undefined; - const dataSql = buildTableSelectSql({ + const result = await fetchTableDataForExport({ databaseType: config.db_type, schema: node.schema, tableName: node.label, columns: queryColumns, - limit: 10000, + executePage: (sql) => api.executeQuery(connectionId, database, sql), }); - const result = await api.executeQuery(node.connectionId, node.database, dataSql); const { buildXlsxWorkbook } = await import("@/lib/xlsxExport"); const workbook = buildXlsxWorkbook({ @@ -1136,7 +1139,7 @@ async function exportDataXlsx() { rows: result.rows, }); await saveBinaryFileContent(workbook, `${node.label}.xlsx`, "Excel", "xlsx"); - toast(result.truncated ? t("grid.exported") + " (truncated)" : t("grid.exported")); + toast(t("grid.exported")); } catch (e: any) { toast(t("grid.exportFailed", { message: e?.message || String(e) }), 5000); } diff --git a/src/lib/tableDataExport.ts b/src/lib/tableDataExport.ts new file mode 100644 index 000000000..663747e9d --- /dev/null +++ b/src/lib/tableDataExport.ts @@ -0,0 +1,50 @@ +import type { DatabaseType, QueryResult } from "../types/database.ts"; +import { buildTableSelectSql } from "./tableSelectSql.ts"; + +export const TABLE_DATA_EXPORT_PAGE_SIZE = 10_000; + +export interface FetchTableDataForExportOptions { + databaseType?: DatabaseType; + schema?: string; + tableName: string; + columns?: string[]; + pageSize?: number; + executePage: (sql: string) => Promise; +} + +export async function fetchTableDataForExport(options: FetchTableDataForExportOptions): Promise { + const pageSize = Math.max(1, options.pageSize ?? TABLE_DATA_EXPORT_PAGE_SIZE); + let offset = 0; + const rows: QueryResult["rows"] = []; + let columns: string[] = []; + let executionTimeMs = 0; + + while (true) { + const sql = buildTableSelectSql({ + databaseType: options.databaseType, + schema: options.schema, + tableName: options.tableName, + columns: options.columns, + limit: pageSize, + offset, + }); + const result = await options.executePage(sql); + if (columns.length === 0) columns = result.columns; + rows.push(...result.rows); + executionTimeMs += result.execution_time_ms ?? 0; + + if (result.rows.length < pageSize) { + return { + columns, + rows, + affected_rows: 0, + execution_time_ms: executionTimeMs, + truncated: false, + session_id: undefined, + has_more: false, + }; + } + + offset += result.rows.length; + } +} diff --git a/tests/tableDataExport.test.ts b/tests/tableDataExport.test.ts new file mode 100644 index 000000000..6608057ae --- /dev/null +++ b/tests/tableDataExport.test.ts @@ -0,0 +1,40 @@ +import { strict as assert } from "node:assert"; +import test from "node:test"; +import { fetchTableDataForExport, TABLE_DATA_EXPORT_PAGE_SIZE } from "../src/lib/tableDataExport.ts"; +import type { QueryResult } from "../src/types/database.ts"; + +function result(rows: QueryResult["rows"]): QueryResult { + return { + columns: ["id"], + rows, + affected_rows: 0, + execution_time_ms: 1, + truncated: false, + has_more: false, + }; +} + +test("fetchTableDataForExport pages past the 10000 row export boundary", async () => { + const sqls: string[] = []; + const pages = [ + result(Array.from({ length: TABLE_DATA_EXPORT_PAGE_SIZE }, (_, index) => [index + 1])), + result([[10_001], [10_002]]), + ]; + + const exported = await fetchTableDataForExport({ + databaseType: "mysql", + tableName: "users", + executePage: async (sql) => { + sqls.push(sql); + return pages.shift() ?? result([]); + }, + }); + + assert.equal(exported.rows.length, TABLE_DATA_EXPORT_PAGE_SIZE + 2); + assert.deepEqual(exported.rows.at(-1), [10_002]); + assert.deepEqual(sqls, [ + "SELECT * FROM `users` LIMIT 10000;", + "SELECT * FROM `users` LIMIT 10000 OFFSET 10000;", + ]); + assert.equal(exported.truncated, false); +});