diff --git a/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts b/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts index 23c33daa3..41f18687d 100644 --- a/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts +++ b/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts @@ -38,12 +38,6 @@ const WINDOWS_JRE_REMOVE_ERROR = [ // Every backend message changed away from hardcoded Chinese, paired with the // key and params it must resolve to. const CASES: { name: string; message: string; key: string; params?: Record }[] = [ - { - name: "XLSX row limit", - message: "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result.", - key: "exportProgress.xlsxRowLimit", - params: { limit: "1,048,575" }, - }, { name: "streaming export unsupported", message: "Streaming export is unsupported for this query. Simplify it or use a supported driver.", @@ -172,7 +166,6 @@ describe("backend error wording is pinned to the Rust sources", () => { const rust = (path: string) => readFileSync(new URL(`../../../../../${path}`, import.meta.url), "utf8"); test.each([ - ["crates/dbx-core/src/query_result_export.rs", "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result."], ["crates/dbx-core/src/query_result_export.rs", "Streaming export is unsupported for this query. Simplify it or use a supported driver."], ["crates/dbx-core/src/query_result_export.rs", "Streaming export needs a result-set session, but this driver returned no session_id."], ["crates/dbx-core/src/agent_service.rs", "Failed to remove the old JRE directory: "], diff --git a/apps/desktop/src/i18n/backend-errors.ts b/apps/desktop/src/i18n/backend-errors.ts index 963056e8d..37f811994 100644 --- a/apps/desktop/src/i18n/backend-errors.ts +++ b/apps/desktop/src/i18n/backend-errors.ts @@ -64,7 +64,6 @@ const patterns: [RegExp, string][] = [ [/^Unsupported SOCKS bound address type: (\d+)$/, "settings.tunnelsSocksUnsupportedAddrType"], // Query result export limits (crates/dbx-core/src/query_result_export.rs) - [/^XLSX supports at most ([\d,]+) data rows\. Use CSV export for the full result\.$/, "exportProgress.xlsxRowLimit"], [/^Streaming export is unsupported for this query\. Simplify it or use a supported driver\.$/, "exportProgress.streamingUnsupported"], [/^Streaming export needs a result-set session, but this driver returned no session_id\.$/, "exportProgress.agentSessionMissing"], @@ -105,7 +104,6 @@ const paramNames: Record = { "settings.tunnelsSocksUnsupportedAuth": "method", "settings.tunnelsSocksConnectRejected": "code", "settings.tunnelsSocksUnsupportedAddrType": "type", - "exportProgress.xlsxRowLimit": "limit", "driverStore.jreDirRemoveFailedWindows": ["path", "error"], "driverStore.jreDirRemoveFailed": ["path", "error"], "driverStore.jreInUseByDrivers": ["jre", "drivers"], diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts index d363bc61c..9841a6b79 100644 --- a/apps/desktop/src/i18n/locales/en.ts +++ b/apps/desktop/src/i18n/locales/en.ts @@ -1447,7 +1447,6 @@ export default { truncatedHint: "Results were truncated after loading {count} rows. Use the footer pagination to browse loaded data; exporting the full result reruns the database query.", }, exportProgress: { - xlsxRowLimit: "XLSX supports at most {limit} data rows. Use CSV export for the full result.", streamingUnsupported: "Streaming export is unsupported for this query. Simplify it or use a supported driver.", agentSessionMissing: "Streaming export needs a result-set session, but this driver returned no session_id.", title: "Exporting Table Data", diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts index dc9fbd2a3..ef9634b10 100644 --- a/apps/desktop/src/i18n/locales/es.ts +++ b/apps/desktop/src/i18n/locales/es.ts @@ -1389,7 +1389,6 @@ export default withEnglishFallback({ xlsxHeaderComment: "Encabezado usando comentarios", }, exportProgress: { - xlsxRowLimit: "XLSX admite como máximo {limit} filas de datos. Use la exportación CSV para obtener el resultado completo.", streamingUnsupported: "La exportación en streaming no es compatible con esta consulta. Simplifíquela o use un controlador compatible.", agentSessionMissing: "La exportación en streaming requiere una sesión de conjunto de resultados, pero este controlador no devolvió session_id.", title: "Exportando datos de la tabla", diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts index 59353aefd..747eba4b9 100644 --- a/apps/desktop/src/i18n/locales/it.ts +++ b/apps/desktop/src/i18n/locales/it.ts @@ -1387,7 +1387,6 @@ export default withEnglishFallback({ xlsxHeaderComment: "Intestazione con commento", }, exportProgress: { - xlsxRowLimit: "XLSX supporta al massimo {limit} righe di dati. Usa l'esportazione CSV per il risultato completo.", streamingUnsupported: "L'esportazione in streaming non è supportata per questa query. Semplificala o usa un driver supportato.", agentSessionMissing: "L'esportazione in streaming richiede una sessione del set di risultati, ma questo driver non ha restituito session_id.", title: "Esportazione Dati Tabella", diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts index 341650145..4375b5d70 100644 --- a/apps/desktop/src/i18n/locales/ja.ts +++ b/apps/desktop/src/i18n/locales/ja.ts @@ -1414,7 +1414,6 @@ export default withEnglishFallback({ emptyStringValue: "空文字", }, exportProgress: { - xlsxRowLimit: "XLSX は最大 {limit} 行のデータに対応しています。完全な結果を得るには CSV エクスポートを使用してください。", streamingUnsupported: "このクエリはストリーミングエクスポートに対応していません。クエリを簡略化するか、対応しているドライバーを使用してください。", agentSessionMissing: "ストリーミングエクスポートには結果セットのセッションが必要ですが、このドライバーは session_id を返しませんでした。", title: "テーブルデータをエクスポート中", diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts index 694c8ca7a..c04dd5af2 100644 --- a/apps/desktop/src/i18n/locales/ko.ts +++ b/apps/desktop/src/i18n/locales/ko.ts @@ -1425,7 +1425,6 @@ export default withEnglishFallback({ truncatedHint: "{count}행 로드 후 결과가 잘렸습니다. 로드된 데이터를 탐색하려면 푸터 페이지 매김을 사용하세요. 전체 결과를 내보내면 데이터베이스 쿼리를 다시 실행합니다.", }, exportProgress: { - xlsxRowLimit: "XLSX는 최대 {limit}개의 데이터 행을 지원합니다. 전체 결과는 CSV 내보내기를 사용하세요.", streamingUnsupported: "이 쿼리는 스트리밍 내보내기를 지원하지 않습니다. 쿼리를 단순화하거나 지원되는 드라이버를 사용하세요.", agentSessionMissing: "스트리밍 내보내기에는 결과 집합 세션이 필요하지, 이 드라이버는 session_id를 반환하지 않았습니다.", title: "테이블 데이터 내보내는 중", diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts index ed8918fef..e9897234f 100644 --- a/apps/desktop/src/i18n/locales/pt-BR.ts +++ b/apps/desktop/src/i18n/locales/pt-BR.ts @@ -1389,7 +1389,6 @@ export default withEnglishFallback({ xlsxHeaderComment: "Cabeçalho usa comentário", }, exportProgress: { - xlsxRowLimit: "O XLSX suporta no máximo {limit} linhas de dados. Use a exportação CSV para obter o resultado completo.", streamingUnsupported: "A exportação em streaming não é compatível com esta consulta. Simplifique-a ou use um driver compatível.", agentSessionMissing: "A exportação em streaming requer uma sessão de conjunto de resultados, mas este driver não retornou session_id.", title: "Exportando Dados da Tabela", diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts index b346deb0c..db5bc28c3 100644 --- a/apps/desktop/src/i18n/locales/zh-CN.ts +++ b/apps/desktop/src/i18n/locales/zh-CN.ts @@ -1448,7 +1448,6 @@ export default withEnglishFallback({ truncatedHint: "结果已截断,已加载前 {count} 行。可通过底部分页浏览已加载数据;导出完整结果时会重新查询数据库。", }, exportProgress: { - xlsxRowLimit: "XLSX 最多支持 {limit} 行数据,请改用 CSV 导出完整结果。", streamingUnsupported: "当前查询暂不支持流式导出,请简化查询或使用受支持的驱动。", agentSessionMissing: "流式导出需要结果集会话,但当前驱动未返回 session_id。", title: "导出表数据", diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts index ac243bd05..2aeaf1a03 100644 --- a/apps/desktop/src/i18n/locales/zh-TW.ts +++ b/apps/desktop/src/i18n/locales/zh-TW.ts @@ -1388,7 +1388,6 @@ export default withEnglishFallback({ xlsxHeaderComment: "表頭使用註解", }, exportProgress: { - xlsxRowLimit: "XLSX 最多支援 {limit} 列資料,請改用 CSV 匯出完整結果。", streamingUnsupported: "目前查詢暫不支援串流匯出,請簡化查詢或使用支援的驅動程式。", agentSessionMissing: "串流匯出需要結果集工作階段,但目前驅動程式未回傳 session_id。", title: "匯出資料表資料", diff --git a/apps/desktop/src/lib/export/xlsxExport.ts b/apps/desktop/src/lib/export/xlsxExport.ts index d19cb8191..413718dea 100644 --- a/apps/desktop/src/lib/export/xlsxExport.ts +++ b/apps/desktop/src/lib/export/xlsxExport.ts @@ -11,6 +11,13 @@ export interface XlsxWorksheetData { numericColumnRightAlign?: boolean; } +interface XlsxWorksheetSegment { + worksheet: XlsxWorksheetData; + sheetName?: string; + rowStart: number; + rowEnd: number; +} + type ZipEntry = { path: string; data: Uint8Array; @@ -20,6 +27,7 @@ type ZipEntry = { const encoder = new TextEncoder(); const CRC_TABLE = buildCrcTable(); +const XLSX_MAX_DATA_ROWS = 1_048_575; function buildCrcTable(): number[] { const table: number[] = []; @@ -84,15 +92,20 @@ function normalizeSheetName(value?: string): string { return name.slice(0, 31); } -function normalizeUniqueSheetNames(sheets: readonly XlsxWorksheetData[]): string[] { +function appendSheetSuffix(base: string, suffix: number): string { + const suffixText = ` (${suffix})`; + const maxBaseLength = Math.max(0, 31 - [...suffixText].length); + return `${[...base].slice(0, maxBaseLength).join("")}${suffixText}`; +} + +function normalizeUniqueSheetNames(sheets: readonly { sheetName?: string }[]): string[] { const names: string[] = []; sheets.forEach((sheet, index) => { const base = normalizeSheetName(sheet.sheetName || `Sheet${index + 1}`); let candidate = base; let suffix = 2; while (names.includes(candidate)) { - const suffixText = ` (${suffix})`; - candidate = `${base.slice(0, 31 - suffixText.length)}${suffixText}`; + candidate = appendSheetSuffix(base, suffix); suffix += 1; } names.push(candidate); @@ -100,10 +113,10 @@ function normalizeUniqueSheetNames(sheets: readonly XlsxWorksheetData[]): string return names; } -function estimateColumnWidths(columns: readonly string[], rows: readonly (readonly XlsxCellValue[])[], columnComments?: readonly (string | null)[]): number[] { +function estimateColumnWidths(columns: readonly string[], rows: readonly (readonly XlsxCellValue[])[], rowStart: number, rowEnd: number, columnComments?: readonly (string | null)[]): number[] { return columns.map((column, colIndex) => { const headerText = columnComments?.[colIndex] || column; - const values = rows.slice(0, 100).map((row) => row[colIndex]); + const values = Array.from({ length: Math.min(100, rowEnd - rowStart) }, (_, index) => rows[rowStart + index]?.[colIndex]); const maxLen = [headerText, ...values.map((value) => (value == null ? "" : String(value)))].map((value) => Math.min(value.length, 60)).reduce((max, length) => Math.max(max, length), 8); return Math.max(10, Math.min(60, maxLen + 2)); }); @@ -143,22 +156,22 @@ function cellXml(value: XlsxCellValue, rowIndex: number, colIndex: number, style return `${escapeXml(String(value))}`; } -function worksheetXml(data: XlsxWorksheetData): string { +function worksheetXml(segment: XlsxWorksheetSegment): string { + const data = segment.worksheet; const columns = data.columns; const rows = data.rows; - const totalRows = rows.length + 1; + const totalRows = segment.rowEnd - segment.rowStart + 1; const range = sheetRange(columns.length, totalRows); - const widths = estimateColumnWidths(columns, rows, data.columnComments); + const widths = estimateColumnWidths(columns, rows, segment.rowStart, segment.rowEnd, data.columnComments); const rightAlignEnabled = data.numericColumnRightAlign !== false; const colsXml = widths.map((width, index) => ``).join(""); const headerXml = `${columns.map((column, index) => cellXml(data.columnComments?.[index] || column, 0, index, 1)).join("")}`; - const bodyXml = rows - .map((row, rowIndex) => { - const excelRowIndex = rowIndex + 2; - const cells = columns.map((_, colIndex) => cellXml(row[colIndex], excelRowIndex - 1, colIndex, numericColumnStyle(data.columnTypes?.[colIndex], rightAlignEnabled), data.columnTypes?.[colIndex])).join(""); - return `${cells}`; - }) - .join(""); + const bodyXml = Array.from({ length: segment.rowEnd - segment.rowStart }, (_, rowIndex) => { + const row = rows[segment.rowStart + rowIndex]!; + const excelRowIndex = rowIndex + 2; + const cells = columns.map((_, colIndex) => cellXml(row[colIndex], excelRowIndex - 1, colIndex, numericColumnStyle(data.columnTypes?.[colIndex], rightAlignEnabled), data.columnTypes?.[colIndex])).join(""); + return `${cells}`; + }).join(""); return ` @@ -171,6 +184,30 @@ function worksheetXml(data: XlsxWorksheetData): string { `; } +function splitWorksheetsForMaxRows(sheets: readonly XlsxWorksheetData[], maxDataRowsPerSheet: number): XlsxWorksheetSegment[] { + const maxRows = Number.isFinite(maxDataRowsPerSheet) ? Math.max(1, Math.floor(maxDataRowsPerSheet)) : XLSX_MAX_DATA_ROWS; + const segments: XlsxWorksheetSegment[] = []; + + for (const worksheet of sheets) { + if (worksheet.rows.length <= maxRows) { + segments.push({ worksheet, sheetName: worksheet.sheetName, rowStart: 0, rowEnd: worksheet.rows.length }); + continue; + } + + const baseName = normalizeSheetName(worksheet.sheetName); + for (let rowStart = 0, chunkIndex = 0; rowStart < worksheet.rows.length; rowStart += maxRows, chunkIndex += 1) { + segments.push({ + worksheet, + sheetName: chunkIndex === 0 ? baseName : appendSheetSuffix(baseName, chunkIndex + 1), + rowStart, + rowEnd: Math.min(rowStart + maxRows, worksheet.rows.length), + }); + } + } + + return segments; +} + function contentTypesXml(sheetCount = 1): string { const worksheetOverrides = Array.from({ length: sheetCount }, (_, index) => ` `).join("\n"); return ` @@ -278,14 +315,19 @@ export function buildXlsxWorkbook(data: XlsxWorksheetData): Uint8Array { } export function buildXlsxWorkbookMulti(sheets: readonly XlsxWorksheetData[]): Uint8Array { + return buildXlsxWorkbookMultiWithMaxRows(sheets, XLSX_MAX_DATA_ROWS); +} + +export function buildXlsxWorkbookMultiWithMaxRows(sheets: readonly XlsxWorksheetData[], maxDataRowsPerSheet: number): Uint8Array { if (sheets.length === 0) throw new Error("At least one worksheet is required"); - const sheetNames = normalizeUniqueSheetNames(sheets); + const segments = splitWorksheetsForMaxRows(sheets, maxDataRowsPerSheet); + const sheetNames = normalizeUniqueSheetNames(segments); return createZip([ - { path: "[Content_Types].xml", content: contentTypesXml(sheets.length) }, + { path: "[Content_Types].xml", content: contentTypesXml(segments.length) }, { path: "_rels/.rels", content: rootRelsXml() }, { path: "xl/workbook.xml", content: workbookXml(sheetNames) }, - { path: "xl/_rels/workbook.xml.rels", content: workbookRelsXml(sheets.length) }, + { path: "xl/_rels/workbook.xml.rels", content: workbookRelsXml(segments.length) }, { path: "xl/styles.xml", content: stylesXml() }, - ...sheets.map((sheet, index) => ({ path: `xl/worksheets/sheet${index + 1}.xml`, content: worksheetXml(sheet) })), + ...segments.map((segment, index) => ({ path: `xl/worksheets/sheet${index + 1}.xml`, content: worksheetXml(segment) })), ]); } diff --git a/crates/dbx-core/src/ai_codex_cli.rs b/crates/dbx-core/src/ai_codex_cli.rs index 9f6e2c46e..29df8d751 100644 --- a/crates/dbx-core/src/ai_codex_cli.rs +++ b/crates/dbx-core/src/ai_codex_cli.rs @@ -984,6 +984,7 @@ pub async fn run_codex_agent( #[cfg(test)] mod tests { + use super::codex_process_env_with_system_proxy; #[cfg(not(windows))] use super::shell_quote; use super::{ @@ -996,7 +997,7 @@ mod tests { DEFAULT_CODEX_MODELS, }; #[cfg(not(windows))] - use super::{codex_process_env, codex_process_env_with_system_proxy, common_executable_dirs, merged_path_with_dir}; + use super::{codex_process_env, common_executable_dirs, merged_path_with_dir}; #[cfg(windows)] use super::{ direct_program_path, first_windows_program_path, program_path_candidates, resolve_codex_command, diff --git a/crates/dbx-core/src/query_result_export.rs b/crates/dbx-core/src/query_result_export.rs index 267b609c5..2cd87ce56 100644 --- a/crates/dbx-core/src/query_result_export.rs +++ b/crates/dbx-core/src/query_result_export.rs @@ -35,8 +35,6 @@ use sqlparser::parser::Parser; use tokio_util::sync::CancellationToken; const AGENT_UNBOUNDED_ROW_LIMIT: usize = i32::MAX as usize; -pub const XLSX_MAX_DATA_ROWS: usize = 1_048_575; -const XLSX_ROW_LIMIT_ERROR: &str = "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result."; const STREAMING_PAGINATION_UNSUPPORTED_ERROR: &str = "Streaming export is unsupported for this query. Simplify it or use a supported driver."; const AGENT_SESSION_MISSING_ERROR: &str = @@ -241,11 +239,7 @@ fn progress( status: ExportStatus, error_message: Option, ) -> TableExportProgress { - let total_rows = request.total_rows.map(|total| { - let format = request.format.to_lowercase(); - let limit = effective_row_limit(&format, request); - limit.map_or(total, |limit| total.min(limit as u64)) - }); + let total_rows = request.total_rows.map(|total| request.row_limit.map_or(total, |limit| total.min(limit as u64))); TableExportProgress { export_id: request.export_id.clone(), table_name: String::new(), @@ -387,16 +381,8 @@ impl SqlInsertWriter { } } -fn effective_row_limit(format: &str, request: &QueryResultExportRequest) -> Option { - if format == "xlsx" { - Some(request.row_limit.map_or(XLSX_MAX_DATA_ROWS, |limit| limit.min(XLSX_MAX_DATA_ROWS))) - } else { - request.row_limit - } -} - -fn xlsx_hard_limit_active(format: &str, request: &QueryResultExportRequest) -> bool { - format == "xlsx" && request.row_limit.is_none_or(|limit| limit > XLSX_MAX_DATA_ROWS) +fn effective_row_limit(request: &QueryResultExportRequest) -> Option { + request.row_limit } fn format_text_export_header(format: &str, columns: &[String]) -> String { @@ -619,18 +605,8 @@ async fn export_query_result_core_inner( } let page_size = request.page_size.max(1); - let effective_row_limit = effective_row_limit(&format, request); - let xlsx_hard_limit_active = xlsx_hard_limit_active(&format, request); - if xlsx_hard_limit_active && request.total_rows.is_some_and(|total| total > XLSX_MAX_DATA_ROWS as u64) { - return Err(XLSX_ROW_LIMIT_ERROR.to_string()); - } - - let agent_max_rows = if xlsx_hard_limit_active { - XLSX_MAX_DATA_ROWS + 1 - } else { - effective_row_limit.unwrap_or(AGENT_UNBOUNDED_ROW_LIMIT) - } - .max(1); + let effective_row_limit = effective_row_limit(request); + let agent_max_rows = effective_row_limit.unwrap_or(AGENT_UNBOUNDED_ROW_LIMIT).max(1); on_progress(progress(request, 0, ExportStatus::Running, None)); @@ -695,11 +671,6 @@ async fn export_query_result_core_inner( break; } let this_page = remaining.map_or(page_size, |rem| rem.min(page_size)).max(1); - let fetch_limit = if xlsx_hard_limit_active && remaining.is_some_and(|rem| rem <= page_size) { - this_page.saturating_add(1) - } else { - this_page - }; let (sql_to_execute, plan_limit, use_agent_result_session) = if let Some(plan) = keyset_plan.as_ref() { ( @@ -710,9 +681,9 @@ async fn export_query_result_core_inner( &request.database_type, &plan.primary_keys, &plan.last_pk_values, - fetch_limit, + this_page, ), - fetch_limit, + this_page, false, ) } else { @@ -720,7 +691,7 @@ async fn export_query_result_core_inner( sql: request.sql.clone(), query_base_sql: request.query_base_sql.clone(), database_type: Some(request.database_type), - pagination: QueryPagination { limit: fetch_limit, offset, session_id: session_id.clone() }, + pagination: QueryPagination { limit: this_page, offset, session_id: session_id.clone() }, use_agent_cursor: request.use_agent_cursor, first_page_uses_actual_sql: true, }); @@ -801,12 +772,6 @@ async fn export_query_result_core_inner( } } let fetched_row_count = result.rows.len(); - if xlsx_hard_limit_active { - let remaining_rows = XLSX_MAX_DATA_ROWS.saturating_sub(rows_exported as usize); - if fetched_row_count > remaining_rows { - return Err(XLSX_ROW_LIMIT_ERROR.to_string()); - } - } if result.rows.len() > this_page { result.rows.truncate(this_page); } @@ -964,10 +929,8 @@ async fn try_export_postgres_query_result_stream( state.touch_pool_activity(&pool_key).await; let _activity_touch = state.pool_activity_touch(&pool_key); - let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request); - let row_limit = effective_row_limit(format, request); - let stream_row_limit = - if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit }; + let row_limit = effective_row_limit(request); + let stream_row_limit = row_limit; let progress_row_interval = request.page_size.max(1) as u64; let mut columns: Vec = Vec::new(); let mut temporal_column_types: Vec = Vec::new(); @@ -1020,9 +983,6 @@ async fn try_export_postgres_query_result_stream( } } crate::db::postgres::PostgresQueryStreamItem::Row(row) => { - if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS { - return Err(XLSX_ROW_LIMIT_ERROR.to_string()); - } let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( &row, &temporal_column_types, @@ -1161,10 +1121,8 @@ async fn try_export_mysql_query_result_stream( crate::query_execution_sql::check_read_only(&request.sql, &name, database_type)?; } - let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request); - let row_limit = effective_row_limit(format, request); - let stream_row_limit = - if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit }; + let row_limit = effective_row_limit(request); + let stream_row_limit = row_limit; let progress_row_interval = request.page_size.max(1) as u64; let mut columns: Vec = Vec::new(); let mut temporal_column_types: Vec = Vec::new(); @@ -1266,9 +1224,6 @@ async fn try_export_mysql_query_result_stream( } } crate::db::mysql::MySqlQueryStreamItem::Row(row) => { - if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS { - return Err(XLSX_ROW_LIMIT_ERROR.to_string()); - } let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( &row, &temporal_column_types, @@ -1427,10 +1382,8 @@ async fn try_export_clickhouse_query_result_stream( state.touch_pool_activity(&pool_key).await; let _activity_touch = state.pool_activity_touch(&pool_key); - let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request); - let row_limit = effective_row_limit(format, request); - let stream_row_limit = - if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit }; + let row_limit = effective_row_limit(request); + let stream_row_limit = row_limit; let progress_row_interval = request.page_size.max(1) as u64; let mut columns: Vec = Vec::new(); let mut temporal_column_types: Vec = Vec::new(); @@ -1484,9 +1437,6 @@ async fn try_export_clickhouse_query_result_stream( } } crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => { - if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS { - return Err(XLSX_ROW_LIMIT_ERROR.to_string()); - } let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( &row, &temporal_column_types, @@ -1600,10 +1550,8 @@ async fn try_export_sqlserver_query_result_stream( state.running_queries.set_pool_key(execution_id, pool_key); } - let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request); - let row_limit = effective_row_limit(format, request); - let stream_row_limit = - if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit }; + let row_limit = effective_row_limit(request); + let stream_row_limit = row_limit; let mut columns: Vec = Vec::new(); let mut temporal_column_types: Vec = Vec::new(); let mut rows_exported = 0_u64; @@ -1667,9 +1615,6 @@ async fn try_export_sqlserver_query_result_stream( } } crate::db::sqlserver::SqlServerStreamItem::Row(row) => { - if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS { - return Err(XLSX_ROW_LIMIT_ERROR.to_string()); - } let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( row, &temporal_column_types, @@ -1865,12 +1810,12 @@ mod tests { #[test] fn csv_unlimited_export_has_no_effective_row_limit() { - assert_eq!(effective_row_limit("csv", &request("csv", None, None)), None); + assert_eq!(effective_row_limit(&request("csv", None, None)), None); } #[test] fn txt_unlimited_export_has_no_effective_row_limit() { - assert_eq!(effective_row_limit("txt", &request("txt", None, None)), None); + assert_eq!(effective_row_limit(&request("txt", None, None)), None); } #[test] @@ -1879,31 +1824,22 @@ mod tests { } #[test] - fn xlsx_unlimited_export_uses_excel_hard_limit() { - assert_eq!(effective_row_limit("xlsx", &request("xlsx", None, None)), Some(XLSX_MAX_DATA_ROWS)); + fn xlsx_no_row_limit_has_no_query_layer_cap() { + // Without the old hard limit, XLSX uses the writer's internal splitting. + assert_eq!(effective_row_limit(&request("xlsx", None, None)), None); } #[test] - fn xlsx_row_limit_caps_to_excel_hard_limit() { - assert_eq!( - effective_row_limit("xlsx", &request("xlsx", Some(XLSX_MAX_DATA_ROWS + 10), None)), - Some(XLSX_MAX_DATA_ROWS) - ); + fn xlsx_user_row_limit_still_respected() { + assert_eq!(effective_row_limit(&request("xlsx", Some(500), None)), Some(500)); } #[test] - fn xlsx_known_total_above_hard_limit_errors_before_export() { - let req = request("xlsx", None, Some(XLSX_MAX_DATA_ROWS as u64 + 1)); - assert!(xlsx_hard_limit_active("xlsx", &req)); - assert!(req.total_rows.is_some_and(|total| total > XLSX_MAX_DATA_ROWS as u64)); - } - - #[test] - fn sql_insert_export_has_no_xlsx_row_cap() { - // SQL format should not be limited by XLSX_MAX_DATA_ROWS. - let req = request("sql", None, None); - assert!(!xlsx_hard_limit_active("sql", &req)); - assert_eq!(effective_row_limit("sql", &req), None); + fn xlsx_total_rows_above_sheet_limit_no_longer_errors() { + // total_rows > 1M no longer triggers a pre-check error; the writer splits. + let req = request("xlsx", None, Some(2_000_000)); + assert!(effective_row_limit(&req).is_none()); + // The function that used to check this (xlsx_hard_limit_active) no longer exists. } #[test] diff --git a/crates/dbx-core/src/xlsx_export.rs b/crates/dbx-core/src/xlsx_export.rs index e9254fa07..363ea74fb 100644 --- a/crates/dbx-core/src/xlsx_export.rs +++ b/crates/dbx-core/src/xlsx_export.rs @@ -4,6 +4,7 @@ use std::io::{Cursor, Seek, Write}; use crate::temporal_format::{excel_temporal_serial, ExcelTemporalKind}; +pub(crate) const XLSX_MAX_DATA_ROWS: usize = 1_048_575; const XLSX_DATE_STYLE: usize = 2; const XLSX_DATETIME_STYLE: usize = 3; const NUMERIC_RIGHT_ALIGN_STYLE: usize = 4; @@ -23,15 +24,110 @@ pub struct XlsxWorksheetData { pub numeric_column_right_align: bool, } +fn normalize_sheet_name(input: Option<&str>) -> String { + let base = input.unwrap_or("Sheet1"); + let name: String = base + .chars() + .map(|ch| match ch { + '[' | ']' | ':' | '*' | '?' | '/' | '\\' => ' ', + _ => ch, + }) + .collect::() + .trim() + .to_string(); + let fallback = if name.is_empty() { "Sheet1" } else { &name }; + fallback.chars().take(31).collect() +} + +/// Allocates unique sheet names for data sheets, avoiding conflicts with +/// pre-reserved trailing sheet names. Data sheets are named sequentially: +/// "Result", "Result (2)", "Result (3)", etc. +struct SheetNameAllocator { + /// The original unsuffixed base name (e.g. "Result"), stored so that + /// [`allocate_next`] always derives continuation names from the original + /// name rather than from an already-de-duplicated first name. + base: String, + data_names: Vec, + trailing_names: Vec, +} + +impl SheetNameAllocator { + fn new(data_sheet_name: Option<&str>, trailing_sheets: &[XlsxWorksheetData]) -> Self { + // Pre-reserve trailing sheet names (normalized, deduplicated) so data + // sheet names never collide with them, and no two trailing sheets share + // the same name. + let mut trailing_names: Vec = Vec::with_capacity(trailing_sheets.len()); + for (index, sheet) in trailing_sheets.iter().enumerate() { + let base = normalize_sheet_name(sheet.sheet_name.as_deref().or(Some(&format!("Sheet{}", index + 1)))); + trailing_names.push(make_unique_name(&base, &trailing_names)); + } + + let base = normalize_sheet_name(data_sheet_name); + // Allocate the first data sheet name, avoiding trailing names. + let reserved: Vec = trailing_names.clone(); + let first = make_unique_name(&base, &reserved); + let data_names = vec![first]; + + Self { base, data_names, trailing_names } + } + + /// Allocate the next data sheet name, avoiding all previously allocated + /// names (both data and trailing). Uses the stored original base name so + /// that continuation names are always "base (2)", "base (3)", ... even + /// when the first data name was de-duplicated. + fn allocate_next(&mut self) { + let mut reserved: Vec = self.trailing_names.clone(); + reserved.extend(self.data_names.iter().cloned()); + let next = make_unique_name(&self.base, &reserved); + self.data_names.push(next); + } + + /// Ordered list: data sheet names first, then trailing sheet names. + fn all_names(&self) -> Vec { + let mut names = self.data_names.clone(); + names.extend(self.trailing_names.clone()); + names + } +} + +fn make_unique_name(base: &str, reserved: &[String]) -> String { + let mut candidate = base.to_string(); + let mut suffix = 2; + while reserved.iter().any(|name| name == &candidate) { + let suffix_text = format!(" ({suffix})"); + let max_base_len = 31usize.saturating_sub(suffix_text.chars().count()); + if max_base_len > 0 { + candidate = format!("{}{}", base.chars().take(max_base_len).collect::(), suffix_text); + } else { + candidate = suffix_text.clone(); + } + suffix += 1; + } + candidate +} + /// Streaming XLSX writer that incrementally writes rows to a ZIP-backed /// workbook. This avoids accumulating all rows in memory before building the /// final file, drastically reducing peak memory for large exports. +/// +/// When the per-sheet data row count reaches [`XLSX_MAX_DATA_ROWS`], the writer +/// automatically closes the current sheet and starts a new one, repeating +/// column headers, frozen panes, and column widths. Trailing sheets (e.g. SQL) +/// are written after all data sheets in [`finish`]. pub struct StreamingXlsxWriter { zip: zip::ZipWriter, columns: Vec, column_types: Vec, next_row_number: usize, + current_data_rows: usize, + max_data_rows_per_sheet: usize, + /// Track the sheet number within the ZIP (1-based). Increments when a new + /// data sheet or trailing sheet is started. + current_sheet_number: usize, + sheet_name_allocator: SheetNameAllocator, trailing_sheets: Vec, + width_cache: Vec, + column_comments: Vec>, date_time_format: Option, numeric_right_align: bool, } @@ -148,7 +244,11 @@ pub(crate) fn start_streaming_xlsx_workbook_with_trailing_sheets( +/// Start a new streaming XLSX workbook with full options. Metadata files +/// ([Content_Types].xml, workbook.xml, styles.xml, etc.) are deferred to +/// [`StreamingXlsxWriter::finish`] so that the final sheet count is known +/// — this supports automatic multi-sheet splitting. +fn start_xlsx_writer_inner( writer: W, sheet_name: Option<&str>, columns: &[String], @@ -157,29 +257,14 @@ pub(crate) fn start_streaming_xlsx_workbook_with_options( trailing_sheets: &[XlsxWorksheetData], date_time_format: Option<&str>, numeric_right_align: bool, + max_data_rows_per_sheet: usize, ) -> Result, String> { - let primary_sheet = XlsxWorksheetData { - sheet_name: sheet_name.map(str::to_string), - columns: columns.to_vec(), - column_types: column_types.to_vec(), - column_comments: column_comments.to_vec(), - rows: Vec::new(), - numeric_column_right_align: numeric_right_align, - }; - let all_sheets = std::iter::once(primary_sheet).chain(trailing_sheets.iter().cloned()).collect::>(); - let sheet_names = normalize_unique_sheet_names(&all_sheets); - let sheet_count = sheet_names.len(); - let widths = estimate_header_widths(columns, column_comments); + let width_cache = estimate_header_widths(columns, column_comments); + let sheet_name_allocator = SheetNameAllocator::new(sheet_name, trailing_sheets); let mut zip = zip::ZipWriter::new(writer); - write_zip_entry(&mut zip, "[Content_Types].xml", &content_types_xml_for_sheet_count(sheet_count))?; - write_zip_entry(&mut zip, "_rels/.rels", root_rels_xml())?; - write_zip_entry(&mut zip, "xl/workbook.xml", &workbook_xml_for_sheets(&sheet_names))?; - write_zip_entry(&mut zip, "xl/_rels/workbook.xml.rels", &workbook_rels_xml_for_sheet_count(sheet_count))?; - write_zip_entry(&mut zip, "xl/styles.xml", &styles_xml(date_time_format))?; - // Begin the sheet1.xml entry with header, frozen pane, column widths and - // the header row. + // Start sheet1 immediately. Metadata files are written in finish(). let options = xlsx_zip_options(); zip.start_file("xl/worksheets/sheet1.xml", options).map_err(|err| err.to_string())?; @@ -194,7 +279,7 @@ pub(crate) fn start_streaming_xlsx_workbook_with_options( "{cols}", "" ), - cols = cols_xml(&widths), + cols = cols_xml(&width_cache), ); zip.write_all(sheet_header.as_bytes()).map_err(|err| err.to_string())?; zip.write_all(header_row_xml(columns, column_comments).as_bytes()).map_err(|err| err.to_string())?; @@ -204,15 +289,72 @@ pub(crate) fn start_streaming_xlsx_workbook_with_options( columns: columns.to_vec(), column_types: column_types.to_vec(), next_row_number: 2, + current_data_rows: 0, + max_data_rows_per_sheet, + current_sheet_number: 1, + sheet_name_allocator, trailing_sheets: trailing_sheets.to_vec(), + width_cache, + column_comments: column_comments.to_vec(), date_time_format: date_time_format.map(str::to_string), numeric_right_align, }) } +pub(crate) fn start_streaming_xlsx_workbook_with_options( + writer: W, + sheet_name: Option<&str>, + columns: &[String], + column_types: &[String], + column_comments: &[Option], + trailing_sheets: &[XlsxWorksheetData], + date_time_format: Option<&str>, + numeric_right_align: bool, +) -> Result, String> { + start_xlsx_writer_inner( + writer, + sheet_name, + columns, + column_types, + column_comments, + trailing_sheets, + date_time_format, + numeric_right_align, + XLSX_MAX_DATA_ROWS, + ) +} + +#[cfg(test)] +pub(crate) fn start_streaming_xlsx_workbook_with_max_rows( + writer: W, + sheet_name: Option<&str>, + columns: &[String], + column_types: &[String], + trailing_sheets: &[XlsxWorksheetData], + max_data_rows_per_sheet: usize, +) -> Result, String> { + start_xlsx_writer_inner( + writer, + sheet_name, + columns, + column_types, + &[], + trailing_sheets, + None, + false, + max_data_rows_per_sheet, + ) +} + impl StreamingXlsxWriter { - /// Append a single data row to the worksheet. + /// Append a single data row to the current worksheet. If the current sheet + /// has reached [`self.max_data_rows_per_sheet`] data rows, this method + /// automatically closes the sheet and opens a new one before writing the row. pub fn write_row(&mut self, row: &[Value]) -> Result<(), String> { + if self.current_data_rows >= self.max_data_rows_per_sheet { + self.finish_current_sheet()?; + self.start_next_data_sheet()?; + } self.zip .write_all( data_row_xml_with_date_time_format( @@ -227,23 +369,95 @@ impl StreamingXlsxWriter { ) .map_err(|err| err.to_string())?; self.next_row_number += 1; + self.current_data_rows += 1; Ok(()) } - /// Finalize the worksheet and close the ZIP archive. Returns the - /// underlying writer so callers can flush / close it as needed. - pub fn finish(mut self) -> Result { + /// Close the currently open data sheet XML: writes ``, + /// `` and ``. + fn finish_current_sheet(&mut self) -> Result<(), String> { let row_count = self.next_row_number.saturating_sub(1); let range = sheet_range(self.columns.len(), row_count); self.zip .write_all(format!("").as_bytes()) + .map_err(|err| err.to_string()) + } + + /// Start a new data sheet, reusing the same header row, column widths and + /// frozen pane from the first sheet. + fn start_next_data_sheet(&mut self) -> Result<(), String> { + self.sheet_name_allocator.allocate_next(); + self.current_sheet_number += 1; + + let options = xlsx_zip_options(); + self.zip + .start_file(format!("xl/worksheets/sheet{}.xml", self.current_sheet_number), options) .map_err(|err| err.to_string())?; - for (index, sheet) in self.trailing_sheets.iter().enumerate() { + + let sheet_header = format!( + concat!( + "", + "", + "", + "", + "", + "", + "{cols}", + "" + ), + cols = cols_xml(&self.width_cache), + ); + self.zip.write_all(sheet_header.as_bytes()).map_err(|err| err.to_string())?; + self.zip + .write_all(header_row_xml(&self.columns, &self.column_comments).as_bytes()) + .map_err(|err| err.to_string())?; + + // Reset row counters for the new sheet. + self.next_row_number = 2; + self.current_data_rows = 0; + Ok(()) + } + + /// Finalize the workbook: close the current data sheet, write trailing + /// sheets, write metadata files, and close the ZIP archive. Returns the + /// underlying writer so callers can flush / close it as needed. + pub fn finish(mut self) -> Result { + // 1. Close the current data sheet. + self.finish_current_sheet()?; + + // 2. Write trailing sheets (e.g. SQL) after all data sheets. + for sheet in &self.trailing_sheets { + self.current_sheet_number += 1; self.zip - .start_file(format!("xl/worksheets/sheet{}.xml", index + 2), xlsx_zip_options()) + .start_file(format!("xl/worksheets/sheet{}.xml", self.current_sheet_number), xlsx_zip_options()) .map_err(|err| err.to_string())?; - self.zip.write_all(worksheet_xml(sheet).as_bytes()).map_err(|err| err.to_string())?; + let segment = WorksheetSegment { + name: sheet.sheet_name.clone(), + columns: &sheet.columns, + column_types: &sheet.column_types, + column_comments: &sheet.column_comments, + rows: &sheet.rows, + numeric_column_right_align: sheet.numeric_column_right_align, + }; + write_worksheet_xml(&mut self.zip, &segment)?; } + + // 3. Write metadata files. These appear AFTER sheet data in the ZIP + // stream, but ZIP readers use the central directory to locate entries + // by name, so the physical ordering is irrelevant. + let sheet_names = self.sheet_name_allocator.all_names(); + let total_sheet_count = sheet_names.len(); + write_zip_entry(&mut self.zip, "[Content_Types].xml", &content_types_xml_for_sheet_count(total_sheet_count))?; + write_zip_entry(&mut self.zip, "_rels/.rels", root_rels_xml())?; + write_zip_entry(&mut self.zip, "xl/workbook.xml", &workbook_xml_for_sheets(&sheet_names))?; + write_zip_entry( + &mut self.zip, + "xl/_rels/workbook.xml.rels", + &workbook_rels_xml_for_sheet_count(total_sheet_count), + )?; + write_zip_entry(&mut self.zip, "xl/styles.xml", &styles_xml(self.date_time_format.as_deref()))?; + + // 4. Finalize the ZIP. self.zip.finish().map_err(|err| err.to_string()) } } @@ -293,21 +507,6 @@ fn sheet_range(column_count: usize, row_count: usize) -> String { format!("A1:{}{}", column_name(column_count - 1), row_count) } -fn normalize_sheet_name(input: Option<&str>) -> String { - let base = input.unwrap_or("Sheet1"); - let name: String = base - .chars() - .map(|ch| match ch { - '[' | ']' | ':' | '*' | '?' | '/' | '\\' => ' ', - _ => ch, - }) - .collect::() - .trim() - .to_string(); - let fallback = if name.is_empty() { "Sheet1" } else { &name }; - fallback.chars().take(31).collect() -} - fn value_text(value: Option<&Value>) -> String { match value { Some(Value::Null) | None => String::new(), @@ -492,58 +691,53 @@ fn typed_cell_xml( cell_xml(value, row_index, col_index, style) } -fn worksheet_xml(data: &XlsxWorksheetData) -> String { - let total_rows = data.rows.len() + 1; - let range = sheet_range(data.columns.len(), total_rows); - let widths = estimate_column_widths(&data.columns, &data.column_comments, &data.rows); +fn write_worksheet_xml(writer: &mut W, segment: &WorksheetSegment) -> Result<(), String> { + let total_rows = segment.rows.len() + 1; + let range = sheet_range(segment.columns.len(), total_rows); + let widths = estimate_column_widths(segment.columns, segment.column_comments, segment.rows); - let cols_xml = widths - .iter() - .enumerate() - .map(|(index, width)| { - format!("", index + 1, index + 1, width) - }) - .collect::(); - - let header_xml = header_row_xml(&data.columns, &data.column_comments); - - let body_xml = data - .rows - .iter() - .enumerate() - .map(|(row_index, row)| { - let excel_row = row_index + 2; - let cells = data - .columns - .iter() - .enumerate() - .map(|(col_index, _)| { - let col_type = data.column_types.get(col_index); - let align_style = numeric_column_style(col_type, data.numeric_column_right_align); - typed_cell_xml(row.get(col_index), col_type, excel_row - 1, col_index, align_style, None) - }) - .collect::(); - format!("{cells}") - }) - .collect::(); - - format!( + writer + .write_all( + format!( concat!( "", "", "", "", "", - "{cols_xml}", - "{header_xml}{body_xml}", - "", - "" + "{cols}", + "" ), range = range, - cols_xml = cols_xml, - header_xml = header_xml, - body_xml = body_xml, + cols = cols_xml(&widths), ) + .as_bytes(), + ) + .map_err(|err| err.to_string())?; + writer + .write_all(header_row_xml(segment.columns, segment.column_comments).as_bytes()) + .map_err(|err| err.to_string())?; + + for (row_index, row) in segment.rows.iter().enumerate() { + let excel_row = row_index + 2; + writer + .write_all( + data_row_xml_with_date_time_format( + excel_row, + segment.columns, + segment.column_types, + row, + None, + segment.numeric_column_right_align, + ) + .as_bytes(), + ) + .map_err(|err| err.to_string())?; + } + + writer + .write_all(format!("").as_bytes()) + .map_err(|err| err.to_string()) } fn content_types_xml_for_sheet_count(sheet_count: usize) -> String { @@ -712,10 +906,27 @@ fn styles_xml(date_time_format: Option<&str>) -> String { ) } -fn normalize_unique_sheet_names(sheets: &[XlsxWorksheetData]) -> Vec { - let mut names = Vec::with_capacity(sheets.len()); - for (index, sheet) in sheets.iter().enumerate() { - let base = normalize_sheet_name(sheet.sheet_name.as_deref().or(Some(&format!("Sheet{}", index + 1)))); +/// A borrow-only view of a worksheet's schema plus a row range, produced by +/// [`split_sheets_for_max_rows`] and consumed by [`write_worksheet_xml`]. Rows are +/// referenced as slices of the original [`XlsxWorksheetData`] rather than +/// deep-copied, so splitting an oversized worksheet into multiple sheets does +/// not duplicate cell data in memory. +struct WorksheetSegment<'a> { + /// Sheet name before cross-sheet deduplication. `None` signals a missing + /// name, which [`normalize_unique_sheet_names`] falls back on with + /// "Sheet{index}" like the original data. + name: Option, + columns: &'a [String], + column_types: &'a [String], + column_comments: &'a [Option], + rows: &'a [Vec], + numeric_column_right_align: bool, +} + +fn normalize_unique_sheet_names(segments: &[WorksheetSegment]) -> Vec { + let mut names = Vec::with_capacity(segments.len()); + for (index, segment) in segments.iter().enumerate() { + let base = normalize_sheet_name(segment.name.as_deref().or(Some(&format!("Sheet{}", index + 1)))); let mut candidate = base.clone(); let mut suffix = 2; while names.iter().any(|name| name == &candidate) { @@ -729,20 +940,86 @@ fn normalize_unique_sheet_names(sheets: &[XlsxWorksheetData]) -> Vec { names } +/// Split any worksheet whose `rows.len() > max_data_rows_per_sheet` into +/// multiple segments, each borrowing the columns / column_types / +/// column_comments / numeric_column_right_align of the original sheet. The +/// first chunk keeps the original sheet name; subsequent chunks get +/// "name (2)", "name (3)", etc. Sheets that do not overflow are passed through +/// as a single segment. No row data is copied. +fn split_sheets_for_max_rows<'a>( + sheets: &'a [XlsxWorksheetData], + max_data_rows_per_sheet: usize, +) -> Vec> { + // `slice::chunks` panics on a zero chunk size. No caller passes 0 today + // (production uses XLSX_MAX_DATA_ROWS), but this is pub(crate)-reachable + // through build_xlsx_workbook_multi_with_max_rows, so guard defensively. + let max_data_rows_per_sheet = max_data_rows_per_sheet.max(1); + let mut expanded = Vec::with_capacity(sheets.len()); + for sheet in sheets { + if sheet.rows.len() <= max_data_rows_per_sheet { + expanded.push(WorksheetSegment { + name: sheet.sheet_name.clone(), + columns: &sheet.columns, + column_types: &sheet.column_types, + column_comments: &sheet.column_comments, + rows: &sheet.rows, + numeric_column_right_align: sheet.numeric_column_right_align, + }); + continue; + } + let base_name = normalize_sheet_name(sheet.sheet_name.as_deref()); + for (chunk_index, chunk) in sheet.rows.chunks(max_data_rows_per_sheet).enumerate() { + let chunk_name = if chunk_index == 0 { + base_name.clone() + } else { + let suffix = chunk_index + 1; + let suffix_text = format!(" ({suffix})"); + let max_base_len = 31usize.saturating_sub(suffix_text.chars().count()); + if max_base_len > 0 { + format!("{}{}", base_name.chars().take(max_base_len).collect::(), suffix_text) + } else { + suffix_text.clone() + } + }; + expanded.push(WorksheetSegment { + name: Some(chunk_name), + columns: &sheet.columns, + column_types: &sheet.column_types, + column_comments: &sheet.column_comments, + rows: chunk, + numeric_column_right_align: sheet.numeric_column_right_align, + }); + } + } + expanded +} + pub fn build_xlsx_workbook(data: &XlsxWorksheetData) -> Result, String> { build_xlsx_workbook_multi(std::slice::from_ref(data)) } pub fn build_xlsx_workbook_multi(sheets: &[XlsxWorksheetData]) -> Result, String> { + build_xlsx_workbook_multi_with_max_rows(sheets, XLSX_MAX_DATA_ROWS) +} + +/// Build an in-memory XLSX workbook with an explicit per-sheet data-row limit. +/// Any worksheet whose data rows exceed `max_data_rows_per_sheet` is split into +/// multiple worksheets (each with a header row, column widths, frozen pane, +/// and autoFilter). +pub(crate) fn build_xlsx_workbook_multi_with_max_rows( + sheets: &[XlsxWorksheetData], + max_data_rows_per_sheet: usize, +) -> Result, String> { if sheets.is_empty() { return Err("At least one worksheet is required".to_string()); } - let sheet_names = normalize_unique_sheet_names(sheets); + let segments = split_sheets_for_max_rows(sheets, max_data_rows_per_sheet); + let sheet_names = normalize_unique_sheet_names(&segments); let files = vec![ - ("[Content_Types].xml", content_types_xml_for_sheet_count(sheets.len())), + ("[Content_Types].xml", content_types_xml_for_sheet_count(segments.len())), ("_rels/.rels", root_rels_xml().to_string()), ("xl/workbook.xml", workbook_xml_for_sheets(&sheet_names)), - ("xl/_rels/workbook.xml.rels", workbook_rels_xml_for_sheet_count(sheets.len())), + ("xl/_rels/workbook.xml.rels", workbook_rels_xml_for_sheet_count(segments.len())), ("xl/styles.xml", styles_xml(None)), ]; @@ -754,9 +1031,9 @@ pub fn build_xlsx_workbook_multi(sheets: &[XlsxWorksheetData]) -> Result zip.start_file(path, options).map_err(|err| err.to_string())?; zip.write_all(content.as_bytes()).map_err(|err| err.to_string())?; } - for (index, sheet) in sheets.iter().enumerate() { + for (index, segment) in segments.iter().enumerate() { zip.start_file(format!("xl/worksheets/sheet{}.xml", index + 1), options).map_err(|err| err.to_string())?; - zip.write_all(worksheet_xml(sheet).as_bytes()).map_err(|err| err.to_string())?; + write_worksheet_xml(&mut zip, segment)?; } let output = zip.finish().map_err(|err| err.to_string())?; @@ -766,14 +1043,35 @@ pub fn build_xlsx_workbook_multi(sheets: &[XlsxWorksheetData]) -> Result #[cfg(test)] mod tests { use super::{ - build_xlsx_workbook, build_xlsx_workbook_multi, is_numeric_column_type, start_streaming_xlsx_workbook, + build_xlsx_workbook, build_xlsx_workbook_multi, build_xlsx_workbook_multi_with_max_rows, + is_numeric_column_type, start_streaming_xlsx_workbook, start_streaming_xlsx_workbook_with_max_rows, start_streaming_xlsx_workbook_with_options, start_streaming_xlsx_workbook_with_trailing_sheets, - XlsxWorksheetData, + write_worksheet_xml, WorksheetSegment, XlsxWorksheetData, }; use calamine::{open_workbook_auto, Reader}; use serde_json::{json, Value}; use std::fs; - use std::io::Read; + use std::io::{Read, Write}; + + #[derive(Default)] + struct WriteStats { + bytes_written: usize, + largest_write: usize, + write_calls: usize, + } + + impl Write for WriteStats { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + self.bytes_written += buffer.len(); + self.largest_write = self.largest_write.max(buffer.len()); + self.write_calls += 1; + Ok(buffer.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } /// Read and decompress a single entry from an in-memory XLSX (ZIP) buffer. fn read_zip_entry(bytes: &[u8], path: &str) -> String { @@ -1207,4 +1505,447 @@ mod tests { assert!(!is_numeric_column_type(Some(&column_type)), "expected non-numeric backend type: {column_type}"); } } + + // ----------------------------------------------------------------------- + // Multi-sheet splitting tests + // ----------------------------------------------------------------------- + + fn build_multi_sheet_xlsx(rows: &[Vec], max_rows: usize) -> Vec { + let path = std::env::temp_dir().join(format!("dbx-multi-split-{}.xlsx", uuid::Uuid::new_v4())); + { + let file = fs::File::create(&path).expect("create temp xlsx"); + let mut writer = start_streaming_xlsx_workbook_with_max_rows( + file, + Some("Result"), + &["id".to_string(), "name".to_string()], + &[], + &[], + max_rows, + ) + .expect("start workbook"); + for row in rows { + writer.write_row(row).expect("write row"); + } + drop(writer.finish().expect("finish workbook")); + } + let data = fs::read(&path).expect("read workbook"); + let _ = fs::remove_file(&path); + data + } + + #[test] + fn splits_rows_across_multiple_sheets() { + // 5 rows, max 2 per sheet -> 3 data sheets + let rows: Vec> = (1..=5).map(|i| vec![json!(i), json!(format!("row_{i}"))]).collect(); + let data = build_multi_sheet_xlsx(&rows, 2); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + assert!(workbook_xml.contains("name=\"Result\"")); + assert!(workbook_xml.contains("name=\"Result (2)\"")); + assert!(workbook_xml.contains("name=\"Result (3)\"")); + + // Verify three data sheets exist + let sheet1 = read_zip_entry(&data, "xl/worksheets/sheet1.xml"); + let sheet2 = read_zip_entry(&data, "xl/worksheets/sheet2.xml"); + let sheet3 = read_zip_entry(&data, "xl/worksheets/sheet3.xml"); + + // Each sheet has header row; rows are numbered starting at 1 + assert!(sheet1.contains("row_1") && sheet1.contains("row_2")); + assert!(sheet2.contains("row_3") && sheet2.contains("row_4")); + assert!(sheet3.contains("row_5")); + + // Each sheet has exactly the header + up to 2 data rows + assert_eq!(sheet1.matches("> = vec![vec![json!(1), json!("Ada")]]; + let data = build_multi_sheet_xlsx(&rows, 2); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + assert!(workbook_xml.contains("name=\"Result\"")); + assert!(!workbook_xml.contains("Result (2)")); + + // Only one sheet exists. + assert!(read_zip_entry(&data, "xl/worksheets/sheet1.xml").contains("Ada")); + // sheet2 should not exist. + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(data)).expect("open xlsx"); + assert!(archive.by_name("xl/worksheets/sheet2.xml").is_err()); + } + + #[test] + fn trailing_sheet_placed_after_data_sheets() { + let path = std::env::temp_dir().join(format!("dbx-trailing-split-{}.xlsx", uuid::Uuid::new_v4())); + { + let file = fs::File::create(&path).expect("create temp xlsx"); + let sql_sheet = XlsxWorksheetData { + sheet_name: Some("SQL".to_string()), + columns: vec!["SQL".to_string()], + column_types: vec![], + column_comments: vec![], + rows: vec![vec![json!("SELECT 1")]], + numeric_column_right_align: false, + }; + let mut writer = start_streaming_xlsx_workbook_with_max_rows( + file, + Some("Result"), + &["id".to_string(), "name".to_string()], + &[], + &[sql_sheet], + 2, + ) + .expect("start workbook"); + // Write 5 rows -> 3 data sheets + 1 trailing = 4 total sheets + for i in 1..=5 { + writer.write_row(&[json!(i), json!(format!("row_{i}"))]).expect("write row"); + } + drop(writer.finish().expect("finish workbook")); + } + let data = fs::read(&path).expect("read workbook"); + let _ = fs::remove_file(&path); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + // SQL sheet should appear last + assert!(workbook_xml.contains("name=\"Result\"")); + assert!(workbook_xml.contains("name=\"Result (2)\"")); + assert!(workbook_xml.contains("name=\"Result (3)\"")); + assert!(workbook_xml.contains("name=\"SQL\"")); + + // SQL is the 4th sheet (sheet4.xml) + let sql_sheet = read_zip_entry(&data, "xl/worksheets/sheet4.xml"); + assert!(sql_sheet.contains("SELECT 1")); + + // All ZIP entries should be Deflate-compressed. + assert_all_entries_deflated(&data); + } + + #[test] + fn varchar_leading_zeros_preserved_across_sheets() { + let path = std::env::temp_dir().join(format!("dbx-leading-zeros-{}.xlsx", uuid::Uuid::new_v4())); + { + let file = fs::File::create(&path).expect("create temp xlsx"); + let mut writer = start_streaming_xlsx_workbook_with_max_rows( + file, + Some("Codes"), + &["code".to_string()], + &["varchar".to_string()], + &[], + 1, + ) + .expect("start workbook"); + writer.write_row(&[json!("00123")]).expect("write row on sheet 1"); + writer.write_row(&[json!("04567")]).expect("write row on sheet 2"); + drop(writer.finish().expect("finish workbook")); + } + let data = fs::read(&path).expect("read workbook"); + let _ = fs::remove_file(&path); + + // Both values must use inlineStr (not numeric ``) to preserve leading zeros. + let sheet2 = read_zip_entry(&data, "xl/worksheets/sheet2.xml"); + assert!(sheet2.contains("t=\"inlineStr\""), "sheet2 should use inlineStr for varchar: {sheet2}"); + assert!(sheet2.contains("04567"), "sheet2 should contain 04567: {sheet2}"); + } + + #[test] + fn exactly_at_limit_does_not_create_extra_sheet() { + // max 2 rows, write exactly 2 rows -> 1 data sheet + let rows: Vec> = vec![vec![json!(1), json!("A")], vec![json!(2), json!("B")]]; + let data = build_multi_sheet_xlsx(&rows, 2); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + assert!(!workbook_xml.contains("Result (2)")); + assert!(read_zip_entry(&data, "xl/worksheets/sheet1.xml").contains("A")); + } + + #[test] + fn deduplicates_identically_named_trailing_sheets() { + let path = std::env::temp_dir().join(format!("dbx-trailing-dedup-{}.xlsx", uuid::Uuid::new_v4())); + { + let file = fs::File::create(&path).expect("create temp xlsx"); + let sql_sheet_a = XlsxWorksheetData { + sheet_name: Some("SQL".to_string()), + columns: vec!["SQL".to_string()], + column_types: vec![], + column_comments: vec![], + rows: vec![vec![json!("SELECT 1")]], + numeric_column_right_align: false, + }; + let sql_sheet_b = XlsxWorksheetData { + sheet_name: Some("SQL".to_string()), + columns: vec!["SQL".to_string()], + column_types: vec![], + column_comments: vec![], + rows: vec![vec![json!("SELECT 2")]], + numeric_column_right_align: false, + }; + let mut writer = start_streaming_xlsx_workbook_with_max_rows( + file, + Some("Result"), + &["id".to_string()], + &[], + &[sql_sheet_a, sql_sheet_b], + 100, + ) + .expect("start workbook"); + writer.write_row(&[json!(42)]).expect("write row"); + drop(writer.finish().expect("finish workbook")); + } + let data = fs::read(&path).expect("read workbook"); + let _ = fs::remove_file(&path); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + assert!(workbook_xml.contains("name=\"SQL\""), "workbook: {workbook_xml}"); + assert!(workbook_xml.contains("name=\"SQL (2)\""), "workbook: {workbook_xml}"); + // No duplicate names — each appears exactly once. + assert_eq!(workbook_xml.matches("name=\"SQL\"").count(), 1, "only one sheet named \"SQL\": {workbook_xml}"); + } + + #[test] + fn streaming_continuation_names_use_original_base_when_first_data_name_collides_with_trailing() { + // Regression: allocate_next used to derive the base from the first + // *allocated* data name, which could already carry a de-dup suffix. + // When the data sheet name ("SQL") collides with a trailing sheet + // named "SQL", the first data name becomes "SQL (2)". The second + // data name should be "SQL (3)" — NOT "SQL (2) (2)". + let path = std::env::temp_dir().join(format!("dbx-collision-base-{}.xlsx", uuid::Uuid::new_v4())); + { + let file = fs::File::create(&path).expect("create temp xlsx"); + let sql_sheet = XlsxWorksheetData { + sheet_name: Some("SQL".to_string()), + columns: vec!["SQL".to_string()], + column_types: vec![], + column_comments: vec![], + rows: vec![vec![json!("SELECT 1")]], + numeric_column_right_align: false, + }; + let mut writer = start_streaming_xlsx_workbook_with_max_rows( + file, + Some("SQL"), + &["id".to_string()], + &[], + &[sql_sheet], + 1, // force split: 2 data rows → 2 data sheets + ) + .expect("start workbook"); + writer.write_row(&[json!(1)]).expect("write row 1"); + writer.write_row(&[json!(2)]).expect("write row 2"); + drop(writer.finish().expect("finish workbook")); + } + let data = fs::read(&path).expect("read workbook"); + let _ = fs::remove_file(&path); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + // Trailing "SQL" stays; data sheets: "SQL (2)", "SQL (3)" (NOT "SQL (2) (2)"). + assert!(workbook_xml.contains("name=\"SQL\""), "trailing SQL sheet must exist: {workbook_xml}"); + assert!(workbook_xml.contains("name=\"SQL (2)\""), "first data sheet must be SQL (2): {workbook_xml}"); + assert!(workbook_xml.contains("name=\"SQL (3)\""), "second data sheet must be SQL (3): {workbook_xml}"); + assert!(!workbook_xml.contains("SQL (2) (2)"), "must not have nested de-dup suffix: {workbook_xml}"); + // Exactly 3 sheets total: tail, data1, data2. + assert_eq!(workbook_xml.matches("name=\"").count(), 3, "expected exactly 3 sheets: {workbook_xml}"); + } + + #[test] + fn build_xlsx_workbook_splits_rows_over_limit() { + // 5 data rows, max 2 per sheet → 3 sheets: "Result", "Result (2)", "Result (3)". + let rows: Vec> = (1..=5).map(|i| vec![json!(i), json!(format!("row_{i}"))]).collect(); + let data = build_xlsx_workbook_multi_with_max_rows( + &[XlsxWorksheetData { + sheet_name: Some("Result".to_string()), + columns: vec!["id".to_string(), "name".to_string()], + column_types: vec![], + column_comments: vec![], + rows, + numeric_column_right_align: false, + }], + 2, + ) + .expect("build workbook"); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + assert!(workbook_xml.contains("name=\"Result\""), "workbook: {workbook_xml}"); + assert!(workbook_xml.contains("name=\"Result (2)\""), "workbook: {workbook_xml}"); + assert!(workbook_xml.contains("name=\"Result (3)\""), "workbook: {workbook_xml}"); + // Exactly 3 sheet entries. + assert_eq!(workbook_xml.matches("name=\"").count(), 3, "expected 3 sheets: {workbook_xml}"); + + let sheet1 = read_zip_entry(&data, "xl/worksheets/sheet1.xml"); + let sheet2 = read_zip_entry(&data, "xl/worksheets/sheet2.xml"); + let sheet3 = read_zip_entry(&data, "xl/worksheets/sheet3.xml"); + + // Each sheet has header row; data rows are split consecutively. + assert!(sheet1.contains("row_1") && sheet1.contains("row_2")); + assert!(!sheet1.contains("row_3")); + assert!(sheet2.contains("row_3") && sheet2.contains("row_4")); + assert!(!sheet2.contains("row_1") && !sheet2.contains("row_5")); + assert!(sheet3.contains("row_5")); + assert!(!sheet3.contains("row_1")); + // 1 header + up to 2 data rows per sheet. + let row_elems = |xml: &str| xml.matches("> = (1..=25_000) + .map(|index| { + vec![ + json!(index), + json!(format!("user_{index}")), + json!(format!("user_{index}@example.com")), + json!(index % 2 == 0), + json!(format!("note-{index:06}-{}", "x".repeat(128))), + ] + }) + .collect(); + let worksheet = XlsxWorksheetData { + sheet_name: Some("Users".to_string()), + columns: vec![ + "id".to_string(), + "name".to_string(), + "email".to_string(), + "active".to_string(), + "notes".to_string(), + ], + column_types: vec![ + "integer".to_string(), + "text".to_string(), + "text".to_string(), + "boolean".to_string(), + "text".to_string(), + ], + column_comments: vec![], + rows, + numeric_column_right_align: true, + }; + let segment = WorksheetSegment { + name: worksheet.sheet_name.clone(), + columns: &worksheet.columns, + column_types: &worksheet.column_types, + column_comments: &worksheet.column_comments, + rows: &worksheet.rows, + numeric_column_right_align: worksheet.numeric_column_right_align, + }; + let mut stats = WriteStats::default(); + + write_worksheet_xml(&mut stats, &segment).expect("write large worksheet"); + + assert!(stats.bytes_written > 10_000_000, "expected realistic worksheet size, got {}", stats.bytes_written); + assert!( + stats.largest_write < 16 * 1024, + "worksheet should be streamed row-by-row, largest write was {}", + stats.largest_write + ); + assert!(stats.write_calls >= worksheet.rows.len(), "expected at least one bounded write per row"); + } + + #[test] + fn build_xlsx_workbook_multi_splits_only_overflowing_sheets() { + // Sheet A: 7 rows, max 3 → "A", "A (2)", "A (3)". + // Sheet B: 2 rows, max 3 → stays "B". Total 4 sheets. + let sheet_a = XlsxWorksheetData { + sheet_name: Some("A".to_string()), + columns: vec!["val".to_string()], + column_types: vec![], + column_comments: vec![], + rows: (0..7).map(|i| vec![json!(i)]).collect(), + numeric_column_right_align: false, + }; + let sheet_b = XlsxWorksheetData { + sheet_name: Some("B".to_string()), + columns: vec!["val".to_string()], + column_types: vec![], + column_comments: vec![], + rows: (100..102).map(|i| vec![json!(i)]).collect(), + numeric_column_right_align: false, + }; + let data = build_xlsx_workbook_multi_with_max_rows(&[sheet_a, sheet_b], 3).expect("build workbook"); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + assert!(workbook_xml.contains("name=\"A\""), "workbook: {workbook_xml}"); + assert!(workbook_xml.contains("name=\"A (2)\""), "workbook: {workbook_xml}"); + assert!(workbook_xml.contains("name=\"A (3)\""), "workbook: {workbook_xml}"); + assert!(workbook_xml.contains("name=\"B\""), "workbook: {workbook_xml}"); + assert_eq!(workbook_xml.matches("name=\"").count(), 4, "expected 4 sheets: {workbook_xml}"); + + // Sheet B should be the 4th entry (after A's 3 chunks). + let sheet_b_xml = read_zip_entry(&data, "xl/worksheets/sheet4.xml"); + assert!(sheet_b_xml.contains(">100<"), "sheet B must contain 100: {sheet_b_xml}"); + assert!(sheet_b_xml.contains(">101<"), "sheet B must contain 101: {sheet_b_xml}"); + + // A's chunks: sheet1 (rows 0-2), sheet2 (rows 3-5), sheet3 (row 6). + let sheet1 = read_zip_entry(&data, "xl/worksheets/sheet1.xml"); + assert_eq!(sheet1.matches("> = vec![vec![json!(1), json!("Ada")]]; + let data = build_xlsx_workbook_multi_with_max_rows( + &[XlsxWorksheetData { + sheet_name: Some("MySheet".to_string()), + columns: vec!["id".to_string(), "name".to_string()], + column_types: vec![], + column_comments: vec![], + rows, + numeric_column_right_align: false, + }], + 100, + ) + .expect("build workbook"); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + assert!(workbook_xml.contains("name=\"MySheet\""), "workbook: {workbook_xml}"); + assert!(!workbook_xml.contains("MySheet (2)"), "should not split: {workbook_xml}"); + assert_eq!(workbook_xml.matches("name=\"").count(), 1, "expected exactly 1 sheet: {workbook_xml}"); + + let sheet = read_zip_entry(&data, "xl/worksheets/sheet1.xml"); + assert!(sheet.contains("Ada"), "sheet should contain data: {sheet}"); + // sheet2 must not exist. + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(data)).expect("open xlsx"); + assert!(archive.by_name("xl/worksheets/sheet2.xml").is_err(), "sheet2 must not exist"); + } + + #[test] + fn build_xlsx_workbook_zero_rows_produces_header_only_sheet() { + let data = build_xlsx_workbook(&XlsxWorksheetData { + sheet_name: Some("Empty".to_string()), + columns: vec!["id".to_string(), "name".to_string()], + column_types: vec![], + column_comments: vec![], + rows: vec![], + numeric_column_right_align: false, + }) + .expect("build workbook"); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + assert!(workbook_xml.contains("name=\"Empty\""), "workbook: {workbook_xml}"); + let sheet = read_zip_entry(&data, "xl/worksheets/sheet1.xml"); + // Header row present, no data rows. + assert!(sheet.contains("id"), "header must contain 'id': {sheet}"); + assert!(sheet.contains("name"), "header must contain 'name': {sheet}"); + // Exactly one element (header row). + let row_count = sheet.matches(" 2 data sheets + let rows: Vec> = (1..=3).map(|i| vec![json!(i), json!(format!("row_{i}"))]).collect(); + let data = build_multi_sheet_xlsx(&rows, 2); + + let workbook_xml = read_zip_entry(&data, "xl/workbook.xml"); + assert!(workbook_xml.contains("name=\"Result\"")); + assert!(workbook_xml.contains("name=\"Result (2)\"")); + + let sheet2 = read_zip_entry(&data, "xl/worksheets/sheet2.xml"); + assert!(sheet2.contains("row_3"), "row 3 should be on sheet 2: {sheet2}"); + } } diff --git a/packages/app-tests/useDataGridExport.test.ts b/packages/app-tests/useDataGridExport.test.ts index cd31002c2..d15a95a2f 100644 --- a/packages/app-tests/useDataGridExport.test.ts +++ b/packages/app-tests/useDataGridExport.test.ts @@ -28,7 +28,7 @@ const dialogMock = vi.hoisted(() => ({ save: vi.fn() })); const toastMock = vi.hoisted(() => vi.fn()); const translateMock = vi.hoisted(() => vi.fn((key: string, params?: Record) => { - if (key === "exportProgress.xlsxRowLimit") return `XLSX 最多支持 ${params?.limit} 行数据,请使用 CSV 导出完整结果。`; + if (key === "exportProgress.streamingUnsupported") return "当前查询暂不支持流式导出,请简化查询或使用受支持的驱动。"; if (key === "grid.exportFailed") return `导出失败:${params?.message}`; return key; }), @@ -397,10 +397,10 @@ test("full query result CSV export streams through the backend without loading a assert.equal(exportProgressState.value.filePath, apiMock.startQueryResultExport.mock.calls[0][0].filePath); }); -test("streaming query result export translates terminal backend errors before the toast", async () => { - const rawMessage = "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result."; +test("streaming query result export translates streaming unsupported error before the toast", async () => { + const rawMessage = "Streaming export is unsupported for this query. Simplify it or use a supported driver."; apiMock.startQueryResultExport.mockImplementationOnce(async (request, onProgress) => { - onProgress({ exportId: request.exportId, tableName: "", rowsExported: 0, totalRows: 1_048_576, status: "Error", errorMessage: rawMessage }); + onProgress({ exportId: request.exportId, tableName: "", rowsExported: 0, totalRows: 0, status: "Error", errorMessage: rawMessage }); throw new Error(rawMessage); }); const { composable, exportProgressState } = buildExportHarness(); @@ -408,7 +408,7 @@ test("streaming query result export translates terminal backend errors before th await composable.exportXlsx(); assert.equal(exportProgressState.value.errorMessage, rawMessage); - assert.deepEqual(toastMock.mock.calls.at(-1), ["导出失败:XLSX 最多支持 1,048,575 行数据,请使用 CSV 导出完整结果。", 5000]); + assert.deepEqual(toastMock.mock.calls.at(-1), ["导出失败:当前查询暂不支持流式导出,请简化查询或使用受支持的驱动。", 5000]); }); test("complete local query result XLSX export does not re-execute the query", async () => { diff --git a/packages/app-tests/xlsxExport.test.ts b/packages/app-tests/xlsxExport.test.ts index e5b1f6f6e..a473b1998 100644 --- a/packages/app-tests/xlsxExport.test.ts +++ b/packages/app-tests/xlsxExport.test.ts @@ -1,8 +1,24 @@ import { strict as assert } from "node:assert"; import { test } from "vitest"; -import { buildXlsxWorkbook, buildXlsxWorkbookMulti } from "../../apps/desktop/src/lib/export/xlsxExport.ts"; +import { buildXlsxWorkbook, buildXlsxWorkbookMulti, buildXlsxWorkbookMultiWithMaxRows } from "../../apps/desktop/src/lib/export/xlsxExport.ts"; import { buildXlsxSqlWorksheet } from "../../apps/desktop/src/lib/export/xlsxSqlSheet.ts"; +function readStoredZipEntry(workbook: Uint8Array, entryPath: string): string { + const view = new DataView(workbook.buffer, workbook.byteOffset, workbook.byteLength); + let offset = 0; + while (offset + 30 <= workbook.length && view.getUint32(offset, true) === 0x04034b50) { + const compressedSize = view.getUint32(offset + 18, true); + const fileNameLength = view.getUint16(offset + 26, true); + const extraLength = view.getUint16(offset + 28, true); + const fileNameStart = offset + 30; + const dataStart = fileNameStart + fileNameLength + extraLength; + const fileName = new TextDecoder().decode(workbook.subarray(fileNameStart, fileNameStart + fileNameLength)); + if (fileName === entryPath) return new TextDecoder().decode(workbook.subarray(dataStart, dataStart + compressedSize)); + offset = dataStart + compressedSize; + } + throw new Error(`Missing ZIP entry: ${entryPath}`); +} + test("builds an xlsx workbook zip with worksheet data", () => { const workbook = buildXlsxWorkbook({ sheetName: "Users", @@ -64,6 +80,60 @@ test("builds a result workbook with a separate SQL worksheet", () => { assert.match(text, /SELECT id, name FROM users WHERE active = true/); }); +test("web in-memory XLSX export splits oversized worksheets", () => { + const workbook = buildXlsxWorkbookMultiWithMaxRows( + [ + { + sheetName: "Result", + columns: ["id", "name"], + rows: [ + [1, "row_1"], + [2, "row_2"], + [3, "row_3"], + [4, "row_4"], + [5, "row_5"], + ], + }, + ], + 2, + ); + + const workbookXml = readStoredZipEntry(workbook, "xl/workbook.xml"); + assert.match(workbookXml, /name="Result"/); + assert.match(workbookXml, /name="Result \(2\)"/); + assert.match(workbookXml, /name="Result \(3\)"/); + + const sheet1 = readStoredZipEntry(workbook, "xl/worksheets/sheet1.xml"); + const sheet2 = readStoredZipEntry(workbook, "xl/worksheets/sheet2.xml"); + const sheet3 = readStoredZipEntry(workbook, "xl/worksheets/sheet3.xml"); + assert.match(sheet1, /row_1/); + assert.match(sheet1, /row_2/); + assert.doesNotMatch(sheet1, /row_3/); + assert.match(sheet2, /row_3/); + assert.match(sheet2, /row_4/); + assert.doesNotMatch(sheet2, /row_5/); + assert.match(sheet3, /row_5/); + assert.equal(sheet1.match(/ { + const rowCount = 20_001; + const rows = Array.from({ length: rowCount }, (_, index) => [index + 1, `user_${index + 1}`, `user_${index + 1}@example.com`, index % 2 === 0, `note-${String(index + 1).padStart(6, "0")}-${"x".repeat(48)}`]); + const workbook = buildXlsxWorkbookMultiWithMaxRows([{ sheetName: "Users", columns: ["id", "name", "email", "active", "notes"], rows }], 10_000); + + const workbookXml = readStoredZipEntry(workbook, "xl/workbook.xml"); + assert.equal(workbookXml.match(/ { const bmpPrefix = "x".repeat(32_766); const longSql = `${bmpPrefix}😀tail`; @@ -130,24 +200,7 @@ test("numeric right-align style is applied consistently across cross-database nu // Ensures the front-end XLSX classifier covers the same cross-database // numeric types as the Rust classifier and the grid (ClickHouse wide // integers, Oracle/Dameng binary floats, SQL Server internal names, etc.). - const columnTypes = [ - "Int16", - "Int32", - "Int64", - "Int128", - "UInt256", - "Decimal128(18, 2)", - "Float16", - "BINARY_FLOAT", - "BINARY_DOUBLE", - "decimaln", - "numericn", - "intn", - "floatn", - "moneyn", - "smallmoneyn", - "varchar(50)", - ]; + const columnTypes = ["Int16", "Int32", "Int64", "Int128", "UInt256", "Decimal128(18, 2)", "Float16", "BINARY_FLOAT", "BINARY_DOUBLE", "decimaln", "numericn", "intn", "floatn", "moneyn", "smallmoneyn", "varchar(50)"]; const workbook = buildXlsxWorkbook({ sheetName: "CrossDb", columns: columnTypes.map((t) => t.toLowerCase()),