fix(export): page table data exports

src/components/sidebar/TreeItem.vue

@@ -88,6 +88,7 @@ import {
 } from "@/lib/databaseCapabilities";
 import { sidebarSelectionCopyAction, treeNodeRowAction, treeNodeRowDoubleClickAction } from "@/lib/treeNodeClick";
 import { formatCsv, formatJson, formatSqlInsert } from "@/lib/exportFormats";
+import { fetchTableDataForExport } from "@/lib/tableDataExport";
 import { buildCreateDatabaseSql, supportsCreateDatabaseCharset } from "@/lib/createDatabaseSql";
 import { buildRenameObjectSql, supportsObjectRename, type RenameableObjectType } from "@/lib/objectRenameSql";
 import { hexToRgba } from "@/lib/color";
@@ -1061,29 +1062,30 @@ async function exportStructure() {
 async function exportData(format: "csv" | "json" | "sql") {
   const node = props.node;
   if (!node.connectionId || !node.database) return;
+  const connectionId = node.connectionId;
+  const database = node.database;
   const config = connectionStore.getConfig(node.connectionId);
   if (!config) return;
 
   try {
-    await connectionStore.ensureConnected(node.connectionId);
+    await connectionStore.ensureConnected(connectionId);
     const qualifiedName =
       isSchemaAware(config.db_type) && node.schema
         ? `${quoteIdent(node.schema)}.${quoteIdent(node.label)}`
         : quoteIdent(node.label);
     const queryColumns =
       config.db_type === "neo4j"
-        ? (await api.getColumns(node.connectionId, node.database, node.schema || node.database, node.label)).map(
+        ? (await api.getColumns(connectionId, database, node.schema || database, node.label)).map(
             (column) => column.name,
           )
         : undefined;
-    const dataSql = buildTableSelectSql({
+    const result = await fetchTableDataForExport({
       databaseType: config.db_type,
       schema: node.schema,
       tableName: node.label,
       columns: queryColumns,
-      limit: 10000,
+      executePage: (sql) => api.executeQuery(connectionId, database, sql),
     });
-    const result = await api.executeQuery(node.connectionId, node.database, dataSql);
 
     let content: string;
     let ext: string;
@@ -1100,7 +1102,7 @@ async function exportData(format: "csv" | "json" | "sql") {
     }
 
     await saveFileContent(content, `${node.label}.${ext}`, ext.toUpperCase(), ext);
-    toast(result.truncated ? t("grid.exported") + " (truncated)" : t("grid.exported"));
+    toast(t("grid.exported"));
   } catch (e: any) {
     toast(t("grid.exportFailed", { message: e?.message || String(e) }), 5000);
   }
@@ -1109,25 +1111,26 @@ async function exportData(format: "csv" | "json" | "sql") {
 async function exportDataXlsx() {
   const node = props.node;
   if (!node.connectionId || !node.database) return;
+  const connectionId = node.connectionId;
+  const database = node.database;
   const config = connectionStore.getConfig(node.connectionId);
   if (!config) return;
 
   try {
-    await connectionStore.ensureConnected(node.connectionId);
+    await connectionStore.ensureConnected(connectionId);
     const queryColumns =
       config.db_type === "neo4j"
-        ? (await api.getColumns(node.connectionId, node.database, node.schema || node.database, node.label)).map(
+        ? (await api.getColumns(connectionId, database, node.schema || database, node.label)).map(
             (column) => column.name,
           )
         : undefined;
-    const dataSql = buildTableSelectSql({
+    const result = await fetchTableDataForExport({
       databaseType: config.db_type,
       schema: node.schema,
       tableName: node.label,
       columns: queryColumns,
-      limit: 10000,
+      executePage: (sql) => api.executeQuery(connectionId, database, sql),
     });
-    const result = await api.executeQuery(node.connectionId, node.database, dataSql);
 
     const { buildXlsxWorkbook } = await import("@/lib/xlsxExport");
     const workbook = buildXlsxWorkbook({
@@ -1136,7 +1139,7 @@ async function exportDataXlsx() {
       rows: result.rows,
     });
     await saveBinaryFileContent(workbook, `${node.label}.xlsx`, "Excel", "xlsx");
-    toast(result.truncated ? t("grid.exported") + " (truncated)" : t("grid.exported"));
+    toast(t("grid.exported"));
   } catch (e: any) {
     toast(t("grid.exportFailed", { message: e?.message || String(e) }), 5000);
   }

src/lib/tableDataExport.ts

@@ -0,0 +1,50 @@
+import type { DatabaseType, QueryResult } from "../types/database.ts";
+import { buildTableSelectSql } from "./tableSelectSql.ts";
+
+export const TABLE_DATA_EXPORT_PAGE_SIZE = 10_000;
+
+export interface FetchTableDataForExportOptions {
+  databaseType?: DatabaseType;
+  schema?: string;
+  tableName: string;
+  columns?: string[];
+  pageSize?: number;
+  executePage: (sql: string) => Promise<QueryResult>;
+}
+
+export async function fetchTableDataForExport(options: FetchTableDataForExportOptions): Promise<QueryResult> {
+  const pageSize = Math.max(1, options.pageSize ?? TABLE_DATA_EXPORT_PAGE_SIZE);
+  let offset = 0;
+  const rows: QueryResult["rows"] = [];
+  let columns: string[] = [];
+  let executionTimeMs = 0;
+
+  while (true) {
+    const sql = buildTableSelectSql({
+      databaseType: options.databaseType,
+      schema: options.schema,
+      tableName: options.tableName,
+      columns: options.columns,
+      limit: pageSize,
+      offset,
+    });
+    const result = await options.executePage(sql);
+    if (columns.length === 0) columns = result.columns;
+    rows.push(...result.rows);
+    executionTimeMs += result.execution_time_ms ?? 0;
+
+    if (result.rows.length < pageSize) {
+      return {
+        columns,
+        rows,
+        affected_rows: 0,
+        execution_time_ms: executionTimeMs,
+        truncated: false,
+        session_id: undefined,
+        has_more: false,
+      };
+    }
+
+    offset += result.rows.length;
+  }
+}

tests/tableDataExport.test.ts

@@ -0,0 +1,40 @@
+import { strict as assert } from "node:assert";
+import test from "node:test";
+import { fetchTableDataForExport, TABLE_DATA_EXPORT_PAGE_SIZE } from "../src/lib/tableDataExport.ts";
+import type { QueryResult } from "../src/types/database.ts";
+
+function result(rows: QueryResult["rows"]): QueryResult {
+  return {
+    columns: ["id"],
+    rows,
+    affected_rows: 0,
+    execution_time_ms: 1,
+    truncated: false,
+    has_more: false,
+  };
+}
+
+test("fetchTableDataForExport pages past the 10000 row export boundary", async () => {
+  const sqls: string[] = [];
+  const pages = [
+    result(Array.from({ length: TABLE_DATA_EXPORT_PAGE_SIZE }, (_, index) => [index + 1])),
+    result([[10_001], [10_002]]),
+  ];
+
+  const exported = await fetchTableDataForExport({
+    databaseType: "mysql",
+    tableName: "users",
+    executePage: async (sql) => {
+      sqls.push(sql);
+      return pages.shift() ?? result([]);
+    },
+  });
+
+  assert.equal(exported.rows.length, TABLE_DATA_EXPORT_PAGE_SIZE + 2);
+  assert.deepEqual(exported.rows.at(-1), [10_002]);
+  assert.deepEqual(sqls, [
+    "SELECT * FROM `users` LIMIT 10000;",
+    "SELECT * FROM `users` LIMIT 10000 OFFSET 10000;",
+  ]);
+  assert.equal(exported.truncated, false);
+});