fix(export): split XLSX sheets and stream worksheet XML
This commit is contained in:
Abeautifulsnow
GitHub
parent
ecac4800c4
commit
c43e8455c4
|
|
@ -38,12 +38,6 @@ const WINDOWS_JRE_REMOVE_ERROR = [
|
|||
// Every backend message changed away from hardcoded Chinese, paired with the
|
||||
// key and params it must resolve to.
|
||||
const CASES: { name: string; message: string; key: string; params?: Record<string, string> }[] = [
|
||||
{
|
||||
name: "XLSX row limit",
|
||||
message: "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result.",
|
||||
key: "exportProgress.xlsxRowLimit",
|
||||
params: { limit: "1,048,575" },
|
||||
},
|
||||
{
|
||||
name: "streaming export unsupported",
|
||||
message: "Streaming export is unsupported for this query. Simplify it or use a supported driver.",
|
||||
|
|
@ -172,7 +166,6 @@ describe("backend error wording is pinned to the Rust sources", () => {
|
|||
const rust = (path: string) => readFileSync(new URL(`../../../../../${path}`, import.meta.url), "utf8");
|
||||
|
||||
test.each([
|
||||
["crates/dbx-core/src/query_result_export.rs", "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result."],
|
||||
["crates/dbx-core/src/query_result_export.rs", "Streaming export is unsupported for this query. Simplify it or use a supported driver."],
|
||||
["crates/dbx-core/src/query_result_export.rs", "Streaming export needs a result-set session, but this driver returned no session_id."],
|
||||
["crates/dbx-core/src/agent_service.rs", "Failed to remove the old JRE directory: "],
|
||||
|
|
|
|||
|
|
@ -64,7 +64,6 @@ const patterns: [RegExp, string][] = [
|
|||
[/^Unsupported SOCKS bound address type: (\d+)$/, "settings.tunnelsSocksUnsupportedAddrType"],
|
||||
|
||||
// Query result export limits (crates/dbx-core/src/query_result_export.rs)
|
||||
[/^XLSX supports at most ([\d,]+) data rows\. Use CSV export for the full result\.$/, "exportProgress.xlsxRowLimit"],
|
||||
[/^Streaming export is unsupported for this query\. Simplify it or use a supported driver\.$/, "exportProgress.streamingUnsupported"],
|
||||
[/^Streaming export needs a result-set session, but this driver returned no session_id\.$/, "exportProgress.agentSessionMissing"],
|
||||
|
||||
|
|
@ -105,7 +104,6 @@ const paramNames: Record<string, string | string[]> = {
|
|||
"settings.tunnelsSocksUnsupportedAuth": "method",
|
||||
"settings.tunnelsSocksConnectRejected": "code",
|
||||
"settings.tunnelsSocksUnsupportedAddrType": "type",
|
||||
"exportProgress.xlsxRowLimit": "limit",
|
||||
"driverStore.jreDirRemoveFailedWindows": ["path", "error"],
|
||||
"driverStore.jreDirRemoveFailed": ["path", "error"],
|
||||
"driverStore.jreInUseByDrivers": ["jre", "drivers"],
|
||||
|
|
|
|||
|
|
@ -1447,7 +1447,6 @@ export default {
|
|||
truncatedHint: "Results were truncated after loading {count} rows. Use the footer pagination to browse loaded data; exporting the full result reruns the database query.",
|
||||
},
|
||||
exportProgress: {
|
||||
xlsxRowLimit: "XLSX supports at most {limit} data rows. Use CSV export for the full result.",
|
||||
streamingUnsupported: "Streaming export is unsupported for this query. Simplify it or use a supported driver.",
|
||||
agentSessionMissing: "Streaming export needs a result-set session, but this driver returned no session_id.",
|
||||
title: "Exporting Table Data",
|
||||
|
|
|
|||
|
|
@ -1389,7 +1389,6 @@ export default withEnglishFallback({
|
|||
xlsxHeaderComment: "Encabezado usando comentarios",
|
||||
},
|
||||
exportProgress: {
|
||||
xlsxRowLimit: "XLSX admite como máximo {limit} filas de datos. Use la exportación CSV para obtener el resultado completo.",
|
||||
streamingUnsupported: "La exportación en streaming no es compatible con esta consulta. Simplifíquela o use un controlador compatible.",
|
||||
agentSessionMissing: "La exportación en streaming requiere una sesión de conjunto de resultados, pero este controlador no devolvió session_id.",
|
||||
title: "Exportando datos de la tabla",
|
||||
|
|
|
|||
|
|
@ -1387,7 +1387,6 @@ export default withEnglishFallback({
|
|||
xlsxHeaderComment: "Intestazione con commento",
|
||||
},
|
||||
exportProgress: {
|
||||
xlsxRowLimit: "XLSX supporta al massimo {limit} righe di dati. Usa l'esportazione CSV per il risultato completo.",
|
||||
streamingUnsupported: "L'esportazione in streaming non è supportata per questa query. Semplificala o usa un driver supportato.",
|
||||
agentSessionMissing: "L'esportazione in streaming richiede una sessione del set di risultati, ma questo driver non ha restituito session_id.",
|
||||
title: "Esportazione Dati Tabella",
|
||||
|
|
|
|||
|
|
@ -1414,7 +1414,6 @@ export default withEnglishFallback({
|
|||
emptyStringValue: "空文字",
|
||||
},
|
||||
exportProgress: {
|
||||
xlsxRowLimit: "XLSX は最大 {limit} 行のデータに対応しています。完全な結果を得るには CSV エクスポートを使用してください。",
|
||||
streamingUnsupported: "このクエリはストリーミングエクスポートに対応していません。クエリを簡略化するか、対応しているドライバーを使用してください。",
|
||||
agentSessionMissing: "ストリーミングエクスポートには結果セットのセッションが必要ですが、このドライバーは session_id を返しませんでした。",
|
||||
title: "テーブルデータをエクスポート中",
|
||||
|
|
|
|||
|
|
@ -1425,7 +1425,6 @@ export default withEnglishFallback({
|
|||
truncatedHint: "{count}행 로드 후 결과가 잘렸습니다. 로드된 데이터를 탐색하려면 푸터 페이지 매김을 사용하세요. 전체 결과를 내보내면 데이터베이스 쿼리를 다시 실행합니다.",
|
||||
},
|
||||
exportProgress: {
|
||||
xlsxRowLimit: "XLSX는 최대 {limit}개의 데이터 행을 지원합니다. 전체 결과는 CSV 내보내기를 사용하세요.",
|
||||
streamingUnsupported: "이 쿼리는 스트리밍 내보내기를 지원하지 않습니다. 쿼리를 단순화하거나 지원되는 드라이버를 사용하세요.",
|
||||
agentSessionMissing: "스트리밍 내보내기에는 결과 집합 세션이 필요하지, 이 드라이버는 session_id를 반환하지 않았습니다.",
|
||||
title: "테이블 데이터 내보내는 중",
|
||||
|
|
|
|||
|
|
@ -1389,7 +1389,6 @@ export default withEnglishFallback({
|
|||
xlsxHeaderComment: "Cabeçalho usa comentário",
|
||||
},
|
||||
exportProgress: {
|
||||
xlsxRowLimit: "O XLSX suporta no máximo {limit} linhas de dados. Use a exportação CSV para obter o resultado completo.",
|
||||
streamingUnsupported: "A exportação em streaming não é compatível com esta consulta. Simplifique-a ou use um driver compatível.",
|
||||
agentSessionMissing: "A exportação em streaming requer uma sessão de conjunto de resultados, mas este driver não retornou session_id.",
|
||||
title: "Exportando Dados da Tabela",
|
||||
|
|
|
|||
|
|
@ -1448,7 +1448,6 @@ export default withEnglishFallback({
|
|||
truncatedHint: "结果已截断，已加载前 {count} 行。可通过底部分页浏览已加载数据；导出完整结果时会重新查询数据库。",
|
||||
},
|
||||
exportProgress: {
|
||||
xlsxRowLimit: "XLSX 最多支持 {limit} 行数据，请改用 CSV 导出完整结果。",
|
||||
streamingUnsupported: "当前查询暂不支持流式导出，请简化查询或使用受支持的驱动。",
|
||||
agentSessionMissing: "流式导出需要结果集会话，但当前驱动未返回 session_id。",
|
||||
title: "导出表数据",
|
||||
|
|
|
|||
|
|
@ -1388,7 +1388,6 @@ export default withEnglishFallback({
|
|||
xlsxHeaderComment: "表頭使用註解",
|
||||
},
|
||||
exportProgress: {
|
||||
xlsxRowLimit: "XLSX 最多支援 {limit} 列資料，請改用 CSV 匯出完整結果。",
|
||||
streamingUnsupported: "目前查詢暫不支援串流匯出，請簡化查詢或使用支援的驅動程式。",
|
||||
agentSessionMissing: "串流匯出需要結果集工作階段，但目前驅動程式未回傳 session_id。",
|
||||
title: "匯出資料表資料",
|
||||
|
|
|
|||
|
|
@ -11,6 +11,13 @@ export interface XlsxWorksheetData {
|
|||
numericColumnRightAlign?: boolean;
|
||||
}
|
||||
|
||||
interface XlsxWorksheetSegment {
|
||||
worksheet: XlsxWorksheetData;
|
||||
sheetName?: string;
|
||||
rowStart: number;
|
||||
rowEnd: number;
|
||||
}
|
||||
|
||||
type ZipEntry = {
|
||||
path: string;
|
||||
data: Uint8Array;
|
||||
|
|
@ -20,6 +27,7 @@ type ZipEntry = {
|
|||
|
||||
const encoder = new TextEncoder();
|
||||
const CRC_TABLE = buildCrcTable();
|
||||
const XLSX_MAX_DATA_ROWS = 1_048_575;
|
||||
|
||||
function buildCrcTable(): number[] {
|
||||
const table: number[] = [];
|
||||
|
|
@ -84,15 +92,20 @@ function normalizeSheetName(value?: string): string {
|
|||
return name.slice(0, 31);
|
||||
}
|
||||
|
||||
function normalizeUniqueSheetNames(sheets: readonly XlsxWorksheetData[]): string[] {
|
||||
function appendSheetSuffix(base: string, suffix: number): string {
|
||||
const suffixText = ` (${suffix})`;
|
||||
const maxBaseLength = Math.max(0, 31 - [...suffixText].length);
|
||||
return `${[...base].slice(0, maxBaseLength).join("")}${suffixText}`;
|
||||
}
|
||||
|
||||
function normalizeUniqueSheetNames(sheets: readonly { sheetName?: string }[]): string[] {
|
||||
const names: string[] = [];
|
||||
sheets.forEach((sheet, index) => {
|
||||
const base = normalizeSheetName(sheet.sheetName || `Sheet${index + 1}`);
|
||||
let candidate = base;
|
||||
let suffix = 2;
|
||||
while (names.includes(candidate)) {
|
||||
const suffixText = ` (${suffix})`;
|
||||
candidate = `${base.slice(0, 31 - suffixText.length)}${suffixText}`;
|
||||
candidate = appendSheetSuffix(base, suffix);
|
||||
suffix += 1;
|
||||
}
|
||||
names.push(candidate);
|
||||
|
|
@ -100,10 +113,10 @@ function normalizeUniqueSheetNames(sheets: readonly XlsxWorksheetData[]): string
|
|||
return names;
|
||||
}
|
||||
|
||||
function estimateColumnWidths(columns: readonly string[], rows: readonly (readonly XlsxCellValue[])[], columnComments?: readonly (string | null)[]): number[] {
|
||||
function estimateColumnWidths(columns: readonly string[], rows: readonly (readonly XlsxCellValue[])[], rowStart: number, rowEnd: number, columnComments?: readonly (string | null)[]): number[] {
|
||||
return columns.map((column, colIndex) => {
|
||||
const headerText = columnComments?.[colIndex] || column;
|
||||
const values = rows.slice(0, 100).map((row) => row[colIndex]);
|
||||
const values = Array.from({ length: Math.min(100, rowEnd - rowStart) }, (_, index) => rows[rowStart + index]?.[colIndex]);
|
||||
const maxLen = [headerText, ...values.map((value) => (value == null ? "" : String(value)))].map((value) => Math.min(value.length, 60)).reduce((max, length) => Math.max(max, length), 8);
|
||||
return Math.max(10, Math.min(60, maxLen + 2));
|
||||
});
|
||||
|
|
@ -143,22 +156,22 @@ function cellXml(value: XlsxCellValue, rowIndex: number, colIndex: number, style
|
|||
return `<c r="${ref}" t="inlineStr"${styleAttr}><is><t>${escapeXml(String(value))}</t></is></c>`;
|
||||
}
|
||||
|
||||
function worksheetXml(data: XlsxWorksheetData): string {
|
||||
function worksheetXml(segment: XlsxWorksheetSegment): string {
|
||||
const data = segment.worksheet;
|
||||
const columns = data.columns;
|
||||
const rows = data.rows;
|
||||
const totalRows = rows.length + 1;
|
||||
const totalRows = segment.rowEnd - segment.rowStart + 1;
|
||||
const range = sheetRange(columns.length, totalRows);
|
||||
const widths = estimateColumnWidths(columns, rows, data.columnComments);
|
||||
const widths = estimateColumnWidths(columns, rows, segment.rowStart, segment.rowEnd, data.columnComments);
|
||||
const rightAlignEnabled = data.numericColumnRightAlign !== false;
|
||||
const colsXml = widths.map((width, index) => `<col min="${index + 1}" max="${index + 1}" width="${width}" customWidth="1"/>`).join("");
|
||||
const headerXml = `<row r="1">${columns.map((column, index) => cellXml(data.columnComments?.[index] || column, 0, index, 1)).join("")}</row>`;
|
||||
const bodyXml = rows
|
||||
.map((row, rowIndex) => {
|
||||
const excelRowIndex = rowIndex + 2;
|
||||
const cells = columns.map((_, colIndex) => cellXml(row[colIndex], excelRowIndex - 1, colIndex, numericColumnStyle(data.columnTypes?.[colIndex], rightAlignEnabled), data.columnTypes?.[colIndex])).join("");
|
||||
return `<row r="${excelRowIndex}">${cells}</row>`;
|
||||
})
|
||||
.join("");
|
||||
const bodyXml = Array.from({ length: segment.rowEnd - segment.rowStart }, (_, rowIndex) => {
|
||||
const row = rows[segment.rowStart + rowIndex]!;
|
||||
const excelRowIndex = rowIndex + 2;
|
||||
const cells = columns.map((_, colIndex) => cellXml(row[colIndex], excelRowIndex - 1, colIndex, numericColumnStyle(data.columnTypes?.[colIndex], rightAlignEnabled), data.columnTypes?.[colIndex])).join("");
|
||||
return `<row r="${excelRowIndex}">${cells}</row>`;
|
||||
}).join("");
|
||||
|
||||
return `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
|
|
@ -171,6 +184,30 @@ function worksheetXml(data: XlsxWorksheetData): string {
|
|||
</worksheet>`;
|
||||
}
|
||||
|
||||
function splitWorksheetsForMaxRows(sheets: readonly XlsxWorksheetData[], maxDataRowsPerSheet: number): XlsxWorksheetSegment[] {
|
||||
const maxRows = Number.isFinite(maxDataRowsPerSheet) ? Math.max(1, Math.floor(maxDataRowsPerSheet)) : XLSX_MAX_DATA_ROWS;
|
||||
const segments: XlsxWorksheetSegment[] = [];
|
||||
|
||||
for (const worksheet of sheets) {
|
||||
if (worksheet.rows.length <= maxRows) {
|
||||
segments.push({ worksheet, sheetName: worksheet.sheetName, rowStart: 0, rowEnd: worksheet.rows.length });
|
||||
continue;
|
||||
}
|
||||
|
||||
const baseName = normalizeSheetName(worksheet.sheetName);
|
||||
for (let rowStart = 0, chunkIndex = 0; rowStart < worksheet.rows.length; rowStart += maxRows, chunkIndex += 1) {
|
||||
segments.push({
|
||||
worksheet,
|
||||
sheetName: chunkIndex === 0 ? baseName : appendSheetSuffix(baseName, chunkIndex + 1),
|
||||
rowStart,
|
||||
rowEnd: Math.min(rowStart + maxRows, worksheet.rows.length),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return segments;
|
||||
}
|
||||
|
||||
function contentTypesXml(sheetCount = 1): string {
|
||||
const worksheetOverrides = Array.from({ length: sheetCount }, (_, index) => ` <Override PartName="/xl/worksheets/sheet${index + 1}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>`).join("\n");
|
||||
return `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
|
|
@ -278,14 +315,19 @@ export function buildXlsxWorkbook(data: XlsxWorksheetData): Uint8Array {
|
|||
}
|
||||
|
||||
export function buildXlsxWorkbookMulti(sheets: readonly XlsxWorksheetData[]): Uint8Array {
|
||||
return buildXlsxWorkbookMultiWithMaxRows(sheets, XLSX_MAX_DATA_ROWS);
|
||||
}
|
||||
|
||||
export function buildXlsxWorkbookMultiWithMaxRows(sheets: readonly XlsxWorksheetData[], maxDataRowsPerSheet: number): Uint8Array {
|
||||
if (sheets.length === 0) throw new Error("At least one worksheet is required");
|
||||
const sheetNames = normalizeUniqueSheetNames(sheets);
|
||||
const segments = splitWorksheetsForMaxRows(sheets, maxDataRowsPerSheet);
|
||||
const sheetNames = normalizeUniqueSheetNames(segments);
|
||||
return createZip([
|
||||
{ path: "[Content_Types].xml", content: contentTypesXml(sheets.length) },
|
||||
{ path: "[Content_Types].xml", content: contentTypesXml(segments.length) },
|
||||
{ path: "_rels/.rels", content: rootRelsXml() },
|
||||
{ path: "xl/workbook.xml", content: workbookXml(sheetNames) },
|
||||
{ path: "xl/_rels/workbook.xml.rels", content: workbookRelsXml(sheets.length) },
|
||||
{ path: "xl/_rels/workbook.xml.rels", content: workbookRelsXml(segments.length) },
|
||||
{ path: "xl/styles.xml", content: stylesXml() },
|
||||
...sheets.map((sheet, index) => ({ path: `xl/worksheets/sheet${index + 1}.xml`, content: worksheetXml(sheet) })),
|
||||
...segments.map((segment, index) => ({ path: `xl/worksheets/sheet${index + 1}.xml`, content: worksheetXml(segment) })),
|
||||
]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -984,6 +984,7 @@ pub async fn run_codex_agent(
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::codex_process_env_with_system_proxy;
|
||||
#[cfg(not(windows))]
|
||||
use super::shell_quote;
|
||||
use super::{
|
||||
|
|
@ -996,7 +997,7 @@ mod tests {
|
|||
DEFAULT_CODEX_MODELS,
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
use super::{codex_process_env, codex_process_env_with_system_proxy, common_executable_dirs, merged_path_with_dir};
|
||||
use super::{codex_process_env, common_executable_dirs, merged_path_with_dir};
|
||||
#[cfg(windows)]
|
||||
use super::{
|
||||
direct_program_path, first_windows_program_path, program_path_candidates, resolve_codex_command,
|
||||
|
|
|
|||
|
|
@ -35,8 +35,6 @@ use sqlparser::parser::Parser;
|
|||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const AGENT_UNBOUNDED_ROW_LIMIT: usize = i32::MAX as usize;
|
||||
pub const XLSX_MAX_DATA_ROWS: usize = 1_048_575;
|
||||
const XLSX_ROW_LIMIT_ERROR: &str = "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result.";
|
||||
const STREAMING_PAGINATION_UNSUPPORTED_ERROR: &str =
|
||||
"Streaming export is unsupported for this query. Simplify it or use a supported driver.";
|
||||
const AGENT_SESSION_MISSING_ERROR: &str =
|
||||
|
|
@ -241,11 +239,7 @@ fn progress(
|
|||
status: ExportStatus,
|
||||
error_message: Option<String>,
|
||||
) -> TableExportProgress {
|
||||
let total_rows = request.total_rows.map(|total| {
|
||||
let format = request.format.to_lowercase();
|
||||
let limit = effective_row_limit(&format, request);
|
||||
limit.map_or(total, |limit| total.min(limit as u64))
|
||||
});
|
||||
let total_rows = request.total_rows.map(|total| request.row_limit.map_or(total, |limit| total.min(limit as u64)));
|
||||
TableExportProgress {
|
||||
export_id: request.export_id.clone(),
|
||||
table_name: String::new(),
|
||||
|
|
@ -387,16 +381,8 @@ impl SqlInsertWriter {
|
|||
}
|
||||
}
|
||||
|
||||
fn effective_row_limit(format: &str, request: &QueryResultExportRequest) -> Option<usize> {
|
||||
if format == "xlsx" {
|
||||
Some(request.row_limit.map_or(XLSX_MAX_DATA_ROWS, |limit| limit.min(XLSX_MAX_DATA_ROWS)))
|
||||
} else {
|
||||
request.row_limit
|
||||
}
|
||||
}
|
||||
|
||||
fn xlsx_hard_limit_active(format: &str, request: &QueryResultExportRequest) -> bool {
|
||||
format == "xlsx" && request.row_limit.is_none_or(|limit| limit > XLSX_MAX_DATA_ROWS)
|
||||
fn effective_row_limit(request: &QueryResultExportRequest) -> Option<usize> {
|
||||
request.row_limit
|
||||
}
|
||||
|
||||
fn format_text_export_header(format: &str, columns: &[String]) -> String {
|
||||
|
|
@ -619,18 +605,8 @@ async fn export_query_result_core_inner(
|
|||
}
|
||||
|
||||
let page_size = request.page_size.max(1);
|
||||
let effective_row_limit = effective_row_limit(&format, request);
|
||||
let xlsx_hard_limit_active = xlsx_hard_limit_active(&format, request);
|
||||
if xlsx_hard_limit_active && request.total_rows.is_some_and(|total| total > XLSX_MAX_DATA_ROWS as u64) {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
|
||||
let agent_max_rows = if xlsx_hard_limit_active {
|
||||
XLSX_MAX_DATA_ROWS + 1
|
||||
} else {
|
||||
effective_row_limit.unwrap_or(AGENT_UNBOUNDED_ROW_LIMIT)
|
||||
}
|
||||
.max(1);
|
||||
let effective_row_limit = effective_row_limit(request);
|
||||
let agent_max_rows = effective_row_limit.unwrap_or(AGENT_UNBOUNDED_ROW_LIMIT).max(1);
|
||||
|
||||
on_progress(progress(request, 0, ExportStatus::Running, None));
|
||||
|
||||
|
|
@ -695,11 +671,6 @@ async fn export_query_result_core_inner(
|
|||
break;
|
||||
}
|
||||
let this_page = remaining.map_or(page_size, |rem| rem.min(page_size)).max(1);
|
||||
let fetch_limit = if xlsx_hard_limit_active && remaining.is_some_and(|rem| rem <= page_size) {
|
||||
this_page.saturating_add(1)
|
||||
} else {
|
||||
this_page
|
||||
};
|
||||
|
||||
let (sql_to_execute, plan_limit, use_agent_result_session) = if let Some(plan) = keyset_plan.as_ref() {
|
||||
(
|
||||
|
|
@ -710,9 +681,9 @@ async fn export_query_result_core_inner(
|
|||
&request.database_type,
|
||||
&plan.primary_keys,
|
||||
&plan.last_pk_values,
|
||||
fetch_limit,
|
||||
this_page,
|
||||
),
|
||||
fetch_limit,
|
||||
this_page,
|
||||
false,
|
||||
)
|
||||
} else {
|
||||
|
|
@ -720,7 +691,7 @@ async fn export_query_result_core_inner(
|
|||
sql: request.sql.clone(),
|
||||
query_base_sql: request.query_base_sql.clone(),
|
||||
database_type: Some(request.database_type),
|
||||
pagination: QueryPagination { limit: fetch_limit, offset, session_id: session_id.clone() },
|
||||
pagination: QueryPagination { limit: this_page, offset, session_id: session_id.clone() },
|
||||
use_agent_cursor: request.use_agent_cursor,
|
||||
first_page_uses_actual_sql: true,
|
||||
});
|
||||
|
|
@ -801,12 +772,6 @@ async fn export_query_result_core_inner(
|
|||
}
|
||||
}
|
||||
let fetched_row_count = result.rows.len();
|
||||
if xlsx_hard_limit_active {
|
||||
let remaining_rows = XLSX_MAX_DATA_ROWS.saturating_sub(rows_exported as usize);
|
||||
if fetched_row_count > remaining_rows {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
}
|
||||
if result.rows.len() > this_page {
|
||||
result.rows.truncate(this_page);
|
||||
}
|
||||
|
|
@ -964,10 +929,8 @@ async fn try_export_postgres_query_result_stream(
|
|||
state.touch_pool_activity(&pool_key).await;
|
||||
let _activity_touch = state.pool_activity_touch(&pool_key);
|
||||
|
||||
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request);
|
||||
let row_limit = effective_row_limit(format, request);
|
||||
let stream_row_limit =
|
||||
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
|
||||
let row_limit = effective_row_limit(request);
|
||||
let stream_row_limit = row_limit;
|
||||
let progress_row_interval = request.page_size.max(1) as u64;
|
||||
let mut columns: Vec<String> = Vec::new();
|
||||
let mut temporal_column_types: Vec<String> = Vec::new();
|
||||
|
|
@ -1020,9 +983,6 @@ async fn try_export_postgres_query_result_stream(
|
|||
}
|
||||
}
|
||||
crate::db::postgres::PostgresQueryStreamItem::Row(row) => {
|
||||
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
||||
&row,
|
||||
&temporal_column_types,
|
||||
|
|
@ -1161,10 +1121,8 @@ async fn try_export_mysql_query_result_stream(
|
|||
crate::query_execution_sql::check_read_only(&request.sql, &name, database_type)?;
|
||||
}
|
||||
|
||||
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request);
|
||||
let row_limit = effective_row_limit(format, request);
|
||||
let stream_row_limit =
|
||||
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
|
||||
let row_limit = effective_row_limit(request);
|
||||
let stream_row_limit = row_limit;
|
||||
let progress_row_interval = request.page_size.max(1) as u64;
|
||||
let mut columns: Vec<String> = Vec::new();
|
||||
let mut temporal_column_types: Vec<String> = Vec::new();
|
||||
|
|
@ -1266,9 +1224,6 @@ async fn try_export_mysql_query_result_stream(
|
|||
}
|
||||
}
|
||||
crate::db::mysql::MySqlQueryStreamItem::Row(row) => {
|
||||
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
||||
&row,
|
||||
&temporal_column_types,
|
||||
|
|
@ -1427,10 +1382,8 @@ async fn try_export_clickhouse_query_result_stream(
|
|||
state.touch_pool_activity(&pool_key).await;
|
||||
let _activity_touch = state.pool_activity_touch(&pool_key);
|
||||
|
||||
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request);
|
||||
let row_limit = effective_row_limit(format, request);
|
||||
let stream_row_limit =
|
||||
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
|
||||
let row_limit = effective_row_limit(request);
|
||||
let stream_row_limit = row_limit;
|
||||
let progress_row_interval = request.page_size.max(1) as u64;
|
||||
let mut columns: Vec<String> = Vec::new();
|
||||
let mut temporal_column_types: Vec<String> = Vec::new();
|
||||
|
|
@ -1484,9 +1437,6 @@ async fn try_export_clickhouse_query_result_stream(
|
|||
}
|
||||
}
|
||||
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
|
||||
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
||||
&row,
|
||||
&temporal_column_types,
|
||||
|
|
@ -1600,10 +1550,8 @@ async fn try_export_sqlserver_query_result_stream(
|
|||
state.running_queries.set_pool_key(execution_id, pool_key);
|
||||
}
|
||||
|
||||
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request);
|
||||
let row_limit = effective_row_limit(format, request);
|
||||
let stream_row_limit =
|
||||
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
|
||||
let row_limit = effective_row_limit(request);
|
||||
let stream_row_limit = row_limit;
|
||||
let mut columns: Vec<String> = Vec::new();
|
||||
let mut temporal_column_types: Vec<String> = Vec::new();
|
||||
let mut rows_exported = 0_u64;
|
||||
|
|
@ -1667,9 +1615,6 @@ async fn try_export_sqlserver_query_result_stream(
|
|||
}
|
||||
}
|
||||
crate::db::sqlserver::SqlServerStreamItem::Row(row) => {
|
||||
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
||||
row,
|
||||
&temporal_column_types,
|
||||
|
|
@ -1865,12 +1810,12 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn csv_unlimited_export_has_no_effective_row_limit() {
|
||||
assert_eq!(effective_row_limit("csv", &request("csv", None, None)), None);
|
||||
assert_eq!(effective_row_limit(&request("csv", None, None)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn txt_unlimited_export_has_no_effective_row_limit() {
|
||||
assert_eq!(effective_row_limit("txt", &request("txt", None, None)), None);
|
||||
assert_eq!(effective_row_limit(&request("txt", None, None)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -1879,31 +1824,22 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn xlsx_unlimited_export_uses_excel_hard_limit() {
|
||||
assert_eq!(effective_row_limit("xlsx", &request("xlsx", None, None)), Some(XLSX_MAX_DATA_ROWS));
|
||||
fn xlsx_no_row_limit_has_no_query_layer_cap() {
|
||||
// Without the old hard limit, XLSX uses the writer's internal splitting.
|
||||
assert_eq!(effective_row_limit(&request("xlsx", None, None)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xlsx_row_limit_caps_to_excel_hard_limit() {
|
||||
assert_eq!(
|
||||
effective_row_limit("xlsx", &request("xlsx", Some(XLSX_MAX_DATA_ROWS + 10), None)),
|
||||
Some(XLSX_MAX_DATA_ROWS)
|
||||
);
|
||||
fn xlsx_user_row_limit_still_respected() {
|
||||
assert_eq!(effective_row_limit(&request("xlsx", Some(500), None)), Some(500));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xlsx_known_total_above_hard_limit_errors_before_export() {
|
||||
let req = request("xlsx", None, Some(XLSX_MAX_DATA_ROWS as u64 + 1));
|
||||
assert!(xlsx_hard_limit_active("xlsx", &req));
|
||||
assert!(req.total_rows.is_some_and(|total| total > XLSX_MAX_DATA_ROWS as u64));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_export_has_no_xlsx_row_cap() {
|
||||
// SQL format should not be limited by XLSX_MAX_DATA_ROWS.
|
||||
let req = request("sql", None, None);
|
||||
assert!(!xlsx_hard_limit_active("sql", &req));
|
||||
assert_eq!(effective_row_limit("sql", &req), None);
|
||||
fn xlsx_total_rows_above_sheet_limit_no_longer_errors() {
|
||||
// total_rows > 1M no longer triggers a pre-check error; the writer splits.
|
||||
let req = request("xlsx", None, Some(2_000_000));
|
||||
assert!(effective_row_limit(&req).is_none());
|
||||
// The function that used to check this (xlsx_hard_limit_active) no longer exists.
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -28,7 +28,7 @@ const dialogMock = vi.hoisted(() => ({ save: vi.fn() }));
|
|||
const toastMock = vi.hoisted(() => vi.fn());
|
||||
const translateMock = vi.hoisted(() =>
|
||||
vi.fn((key: string, params?: Record<string, unknown>) => {
|
||||
if (key === "exportProgress.xlsxRowLimit") return `XLSX 最多支持 ${params?.limit} 行数据，请使用 CSV 导出完整结果。`;
|
||||
if (key === "exportProgress.streamingUnsupported") return "当前查询暂不支持流式导出，请简化查询或使用受支持的驱动。";
|
||||
if (key === "grid.exportFailed") return `导出失败：${params?.message}`;
|
||||
return key;
|
||||
}),
|
||||
|
|
@ -397,10 +397,10 @@ test("full query result CSV export streams through the backend without loading a
|
|||
assert.equal(exportProgressState.value.filePath, apiMock.startQueryResultExport.mock.calls[0][0].filePath);
|
||||
});
|
||||
|
||||
test("streaming query result export translates terminal backend errors before the toast", async () => {
|
||||
const rawMessage = "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result.";
|
||||
test("streaming query result export translates streaming unsupported error before the toast", async () => {
|
||||
const rawMessage = "Streaming export is unsupported for this query. Simplify it or use a supported driver.";
|
||||
apiMock.startQueryResultExport.mockImplementationOnce(async (request, onProgress) => {
|
||||
onProgress({ exportId: request.exportId, tableName: "", rowsExported: 0, totalRows: 1_048_576, status: "Error", errorMessage: rawMessage });
|
||||
onProgress({ exportId: request.exportId, tableName: "", rowsExported: 0, totalRows: 0, status: "Error", errorMessage: rawMessage });
|
||||
throw new Error(rawMessage);
|
||||
});
|
||||
const { composable, exportProgressState } = buildExportHarness();
|
||||
|
|
@ -408,7 +408,7 @@ test("streaming query result export translates terminal backend errors before th
|
|||
await composable.exportXlsx();
|
||||
|
||||
assert.equal(exportProgressState.value.errorMessage, rawMessage);
|
||||
assert.deepEqual(toastMock.mock.calls.at(-1), ["导出失败：XLSX 最多支持 1,048,575 行数据，请使用 CSV 导出完整结果。", 5000]);
|
||||
assert.deepEqual(toastMock.mock.calls.at(-1), ["导出失败：当前查询暂不支持流式导出，请简化查询或使用受支持的驱动。", 5000]);
|
||||
});
|
||||
|
||||
test("complete local query result XLSX export does not re-execute the query", async () => {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,24 @@
|
|||
import { strict as assert } from "node:assert";
|
||||
import { test } from "vitest";
|
||||
import { buildXlsxWorkbook, buildXlsxWorkbookMulti } from "../../apps/desktop/src/lib/export/xlsxExport.ts";
|
||||
import { buildXlsxWorkbook, buildXlsxWorkbookMulti, buildXlsxWorkbookMultiWithMaxRows } from "../../apps/desktop/src/lib/export/xlsxExport.ts";
|
||||
import { buildXlsxSqlWorksheet } from "../../apps/desktop/src/lib/export/xlsxSqlSheet.ts";
|
||||
|
||||
function readStoredZipEntry(workbook: Uint8Array, entryPath: string): string {
|
||||
const view = new DataView(workbook.buffer, workbook.byteOffset, workbook.byteLength);
|
||||
let offset = 0;
|
||||
while (offset + 30 <= workbook.length && view.getUint32(offset, true) === 0x04034b50) {
|
||||
const compressedSize = view.getUint32(offset + 18, true);
|
||||
const fileNameLength = view.getUint16(offset + 26, true);
|
||||
const extraLength = view.getUint16(offset + 28, true);
|
||||
const fileNameStart = offset + 30;
|
||||
const dataStart = fileNameStart + fileNameLength + extraLength;
|
||||
const fileName = new TextDecoder().decode(workbook.subarray(fileNameStart, fileNameStart + fileNameLength));
|
||||
if (fileName === entryPath) return new TextDecoder().decode(workbook.subarray(dataStart, dataStart + compressedSize));
|
||||
offset = dataStart + compressedSize;
|
||||
}
|
||||
throw new Error(`Missing ZIP entry: ${entryPath}`);
|
||||
}
|
||||
|
||||
test("builds an xlsx workbook zip with worksheet data", () => {
|
||||
const workbook = buildXlsxWorkbook({
|
||||
sheetName: "Users",
|
||||
|
|
@ -64,6 +80,60 @@ test("builds a result workbook with a separate SQL worksheet", () => {
|
|||
assert.match(text, /SELECT id, name FROM users WHERE active = true/);
|
||||
});
|
||||
|
||||
test("web in-memory XLSX export splits oversized worksheets", () => {
|
||||
const workbook = buildXlsxWorkbookMultiWithMaxRows(
|
||||
[
|
||||
{
|
||||
sheetName: "Result",
|
||||
columns: ["id", "name"],
|
||||
rows: [
|
||||
[1, "row_1"],
|
||||
[2, "row_2"],
|
||||
[3, "row_3"],
|
||||
[4, "row_4"],
|
||||
[5, "row_5"],
|
||||
],
|
||||
},
|
||||
],
|
||||
2,
|
||||
);
|
||||
|
||||
const workbookXml = readStoredZipEntry(workbook, "xl/workbook.xml");
|
||||
assert.match(workbookXml, /name="Result"/);
|
||||
assert.match(workbookXml, /name="Result \(2\)"/);
|
||||
assert.match(workbookXml, /name="Result \(3\)"/);
|
||||
|
||||
const sheet1 = readStoredZipEntry(workbook, "xl/worksheets/sheet1.xml");
|
||||
const sheet2 = readStoredZipEntry(workbook, "xl/worksheets/sheet2.xml");
|
||||
const sheet3 = readStoredZipEntry(workbook, "xl/worksheets/sheet3.xml");
|
||||
assert.match(sheet1, /row_1/);
|
||||
assert.match(sheet1, /row_2/);
|
||||
assert.doesNotMatch(sheet1, /row_3/);
|
||||
assert.match(sheet2, /row_3/);
|
||||
assert.match(sheet2, /row_4/);
|
||||
assert.doesNotMatch(sheet2, /row_5/);
|
||||
assert.match(sheet3, /row_5/);
|
||||
assert.equal(sheet1.match(/<row r="/g)?.length, 3);
|
||||
assert.equal(sheet2.match(/<row r="/g)?.length, 3);
|
||||
assert.equal(sheet3.match(/<row r="/g)?.length, 2);
|
||||
});
|
||||
|
||||
test("web in-memory XLSX export handles a realistic large result", () => {
|
||||
const rowCount = 20_001;
|
||||
const rows = Array.from({ length: rowCount }, (_, index) => [index + 1, `user_${index + 1}`, `user_${index + 1}@example.com`, index % 2 === 0, `note-${String(index + 1).padStart(6, "0")}-${"x".repeat(48)}`]);
|
||||
const workbook = buildXlsxWorkbookMultiWithMaxRows([{ sheetName: "Users", columns: ["id", "name", "email", "active", "notes"], rows }], 10_000);
|
||||
|
||||
const workbookXml = readStoredZipEntry(workbook, "xl/workbook.xml");
|
||||
assert.equal(workbookXml.match(/<sheet /g)?.length, 3);
|
||||
assert.match(workbookXml, /name="Users \(3\)"/);
|
||||
assert.match(readStoredZipEntry(workbook, "xl/worksheets/sheet1.xml"), /user_10000@example\.com/);
|
||||
assert.match(readStoredZipEntry(workbook, "xl/worksheets/sheet2.xml"), /user_20000@example\.com/);
|
||||
const finalSheet = readStoredZipEntry(workbook, "xl/worksheets/sheet3.xml");
|
||||
assert.match(finalSheet, /user_20001@example\.com/);
|
||||
assert.equal(finalSheet.match(/<row r="/g)?.length, 2);
|
||||
assert.ok(workbook.byteLength > 5_000_000);
|
||||
});
|
||||
|
||||
test("maps multiple result statements and splits SQL at the Excel cell limit", () => {
|
||||
const bmpPrefix = "x".repeat(32_766);
|
||||
const longSql = `${bmpPrefix}😀tail`;
|
||||
|
|
@ -130,24 +200,7 @@ test("numeric right-align style is applied consistently across cross-database nu
|
|||
// Ensures the front-end XLSX classifier covers the same cross-database
|
||||
// numeric types as the Rust classifier and the grid (ClickHouse wide
|
||||
// integers, Oracle/Dameng binary floats, SQL Server internal names, etc.).
|
||||
const columnTypes = [
|
||||
"Int16",
|
||||
"Int32",
|
||||
"Int64",
|
||||
"Int128",
|
||||
"UInt256",
|
||||
"Decimal128(18, 2)",
|
||||
"Float16",
|
||||
"BINARY_FLOAT",
|
||||
"BINARY_DOUBLE",
|
||||
"decimaln",
|
||||
"numericn",
|
||||
"intn",
|
||||
"floatn",
|
||||
"moneyn",
|
||||
"smallmoneyn",
|
||||
"varchar(50)",
|
||||
];
|
||||
const columnTypes = ["Int16", "Int32", "Int64", "Int128", "UInt256", "Decimal128(18, 2)", "Float16", "BINARY_FLOAT", "BINARY_DOUBLE", "decimaln", "numericn", "intn", "floatn", "moneyn", "smallmoneyn", "varchar(50)"];
|
||||
const workbook = buildXlsxWorkbook({
|
||||
sheetName: "CrossDb",
|
||||
columns: columnTypes.map((t) => t.toLowerCase()),
|
||||
|
|
|
|||
Loading…
Reference in New Issue