diff --git a/apps/desktop/src/components/editor/EditorSettingsDialog.vue b/apps/desktop/src/components/editor/EditorSettingsDialog.vue index 753368989..ea0255986 100644 --- a/apps/desktop/src/components/editor/EditorSettingsDialog.vue +++ b/apps/desktop/src/components/editor/EditorSettingsDialog.vue @@ -112,6 +112,7 @@ import { DEFAULT_WEB_DAV_AUTO_UPLOAD_INTERVAL_MINUTES, DEFAULT_WEB_DAV_REMOTE_PA import { apiUrl } from "@/lib/common/webPath"; import { DEFAULT_UI_FONT_FAMILY, SYSTEM_UI_FONT_FAMILY } from "@/lib/app/appFonts"; import { buildAppSupportInfoRows, formatAppSupportInfoForClipboard, type AppSupportInfoLabels } from "@/lib/app/supportInfo"; +import { DateTimePatterns, normalizeSupportedDateTimePattern } from "@/lib/dataGrid/columnFormatter"; const { t } = useI18n(); const { toast } = useToast(); @@ -340,6 +341,9 @@ const editSidebarHiddenTablePrefixes = ref(settingsStore.editorSettings.sidebarH const editSidebarHideTableComments = ref(settingsStore.editorSettings.sidebarHideTableComments); const editSidebarAllowHorizontalScroll = ref(settingsStore.editorSettings.sidebarAllowHorizontalScroll); const editExportBatchSize = ref(settingsStore.editorSettings.exportBatchSize); +const editGlobalDateTimeDisplayFormat = ref(settingsStore.editorSettings.globalDateTimeDisplayFormat); +const editGlobalDateTimeExportFormat = ref(settingsStore.editorSettings.globalDateTimeExportFormat); +const editGlobalDateTimeImportFormat = ref(settingsStore.editorSettings.globalDateTimeImportFormat); const editExportRowLimitEnabled = ref(settingsStore.editorSettings.exportRowLimitEnabled); const editExportRowLimit = ref(settingsStore.editorSettings.exportRowLimit); const editQueryExportKeysetOptimizationEnabled = ref(settingsStore.editorSettings.queryExportKeysetOptimizationEnabled); @@ -427,6 +431,9 @@ function currentEditorSettingsDraft(): EditorSettingsDraft { sidebarAllowHorizontalScroll: editSidebarAllowHorizontalScroll.value, sidebarHiddenTablePrefixes: normalizeSidebarHiddenTablePrefixes(editSidebarHiddenTablePrefixes.value), exportBatchSize: editExportBatchSize.value, + globalDateTimeDisplayFormat: editGlobalDateTimeDisplayFormat.value, + globalDateTimeExportFormat: editGlobalDateTimeExportFormat.value, + globalDateTimeImportFormat: editGlobalDateTimeImportFormat.value, exportRowLimitEnabled: editExportRowLimitEnabled.value, exportRowLimit: editExportRowLimit.value, queryExportKeysetOptimizationEnabled: editQueryExportKeysetOptimizationEnabled.value, @@ -704,6 +711,9 @@ function syncEditorSettingsDraftFromStore() { editSidebarHideTableComments.value = settingsStore.editorSettings.sidebarHideTableComments; editSidebarAllowHorizontalScroll.value = settingsStore.editorSettings.sidebarAllowHorizontalScroll; editExportBatchSize.value = settingsStore.editorSettings.exportBatchSize; + editGlobalDateTimeDisplayFormat.value = settingsStore.editorSettings.globalDateTimeDisplayFormat; + editGlobalDateTimeExportFormat.value = settingsStore.editorSettings.globalDateTimeExportFormat; + editGlobalDateTimeImportFormat.value = settingsStore.editorSettings.globalDateTimeImportFormat; editExportRowLimitEnabled.value = settingsStore.editorSettings.exportRowLimitEnabled; editExportRowLimit.value = settingsStore.editorSettings.exportRowLimit; editQueryExportKeysetOptimizationEnabled.value = settingsStore.editorSettings.queryExportKeysetOptimizationEnabled; @@ -910,6 +920,9 @@ function resetDefaultsForTab(tab: SettingsCategory) { editDuckDbWorkerMaxProcesses.value = DEFAULT_DESKTOP_SETTINGS.duckdb_worker_max_processes; editTableColumnTemplateRows.value = tableColumnTemplateRowsFromSettings(DEFAULT_EDITOR_SETTINGS.tableColumnTemplateFields); editExportBatchSize.value = DEFAULT_EDITOR_SETTINGS.exportBatchSize; + editGlobalDateTimeDisplayFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeDisplayFormat; + editGlobalDateTimeExportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeExportFormat; + editGlobalDateTimeImportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeImportFormat; editExportRowLimitEnabled.value = DEFAULT_EDITOR_SETTINGS.exportRowLimitEnabled; editExportRowLimit.value = DEFAULT_EDITOR_SETTINGS.exportRowLimit; editQueryExportKeysetOptimizationEnabled.value = DEFAULT_EDITOR_SETTINGS.queryExportKeysetOptimizationEnabled; @@ -977,6 +990,9 @@ function resetAllDefaults() { editSidebarAllowHorizontalScroll.value = DEFAULT_EDITOR_SETTINGS.sidebarAllowHorizontalScroll; editSidebarHiddenTablePrefixes.value = DEFAULT_EDITOR_SETTINGS.sidebarHiddenTablePrefixes.join("\n"); editExportBatchSize.value = DEFAULT_EDITOR_SETTINGS.exportBatchSize; + editGlobalDateTimeDisplayFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeDisplayFormat; + editGlobalDateTimeExportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeExportFormat; + editGlobalDateTimeImportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeImportFormat; editExportRowLimitEnabled.value = DEFAULT_EDITOR_SETTINGS.exportRowLimitEnabled; editExportRowLimit.value = DEFAULT_EDITOR_SETTINGS.exportRowLimit; editQueryExportKeysetOptimizationEnabled.value = DEFAULT_EDITOR_SETTINGS.queryExportKeysetOptimizationEnabled; @@ -3590,6 +3606,62 @@ onUnmounted(cleanupPreviewEditor); +
+
{{ t("settings.dateTimeSection") }}
+
+
+ +

{{ t("settings.globalDateTimeDisplayFormatDescription") }}

+
+ +
+ +

{{ t("settings.globalDateTimeExportFormatDescription") }}

+
+ +
+ +

{{ t("settings.globalDateTimeImportFormatDescription") }}

+
+ +
+
+ + +
{{ t("settings.exportSection") }}
diff --git a/apps/desktop/src/components/grid/DataGrid.vue b/apps/desktop/src/components/grid/DataGrid.vue index eee23d31c..40127bec1 100644 --- a/apps/desktop/src/components/grid/DataGrid.vue +++ b/apps/desktop/src/components/grid/DataGrid.vue @@ -963,7 +963,10 @@ function columnFormatter(columnIndex: number): ColumnFormatterConfig | undefined const column = props.result.columns[columnIndex]; if (!column) return undefined; const key = formatterKeyForColumn(column); - return key ? resolveColumnFormatter(settingsStore.editorSettings.columnFormatters[key], settingsStore.editorSettings.customColumnFormatters) : undefined; + return resolveColumnFormatter(key ? settingsStore.editorSettings.columnFormatters[key] : undefined, settingsStore.editorSettings.customColumnFormatters, { + pattern: settingsStore.editorSettings.globalDateTimeDisplayFormat, + columnType: props.result.column_types?.[columnIndex] ?? tableColumnForGridColumn(columnIndex)?.data_type, + }); } function savedColumnFormatter(columnIndex: number): ColumnFormatterConfig | undefined { diff --git a/apps/desktop/src/components/import/TableImportDialog.vue b/apps/desktop/src/components/import/TableImportDialog.vue index eaa39ee39..95bf8f433 100644 --- a/apps/desktop/src/components/import/TableImportDialog.vue +++ b/apps/desktop/src/components/import/TableImportDialog.vue @@ -11,6 +11,7 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@ import { SearchableSelect } from "@/components/ui/searchable-select"; import { AlertTriangle, ArrowLeft, ArrowRight, Check, CheckCircle2, FileJson, FileSpreadsheet, FileText, FileUp, Loader2, RefreshCw, Square, Upload, X } from "@lucide/vue"; import { useConnectionStore } from "@/stores/connectionStore"; +import { useSettingsStore } from "@/stores/settingsStore"; import { useToast } from "@/composables/useToast"; import { autoMapImportColumns, nextTableImportWizardStep, previousTableImportWizardStep, requiredImportTargetColumns, suggestImportTargetDataTypes, validateImportMappings, type TableImportWizardStep } from "@/lib/table/tableImport"; import { getDataTypeOptions } from "@/lib/table/tableStructureEditorState"; @@ -20,6 +21,7 @@ import * as api from "@/lib/backend/api"; const { t } = useI18n(); const store = useConnectionStore(); +const settingsStore = useSettingsStore(); const { toast } = useToast(); const open = defineModel("open", { default: false }); @@ -625,6 +627,7 @@ async function startImport() { mode: targetMode.value === "create" ? "append" : importMode.value, createTable: targetMode.value === "create", batchSize: Math.max(1, Number(batchSize.value) || 500), + dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined, }, (nextProgress) => { progress.value = nextProgress; @@ -700,6 +703,7 @@ async function startBatchImport() { mode: "append", createTable: true, batchSize: Math.max(1, Number(batchSize.value) || 500), + dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined, }, (nextProgress) => { task.rowsImported = nextProgress.rowsImported; diff --git a/apps/desktop/src/composables/useDataGridExport.ts b/apps/desktop/src/composables/useDataGridExport.ts index f6cc3d157..cd5a7778c 100644 --- a/apps/desktop/src/composables/useDataGridExport.ts +++ b/apps/desktop/src/composables/useDataGridExport.ts @@ -18,6 +18,7 @@ import type { DatabaseType, QueryResult } from "@/types/database"; import type { QueryResultExportRequest } from "@/lib/backend/api"; import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing"; import { buildXlsxSqlWorksheet } from "@/lib/export/xlsxSqlSheet"; +import { formatTemporalRowsForExport } from "@/lib/dataGrid/columnFormatter"; /** * Format metadata for backend table exports. Each entry maps a format key @@ -202,10 +203,15 @@ export function useDataGridExport(options: UseDataGridExportOptions) { return displayItems.value.filter((item) => rowIdSet.has(item.id) && !item.isDraft); } - async function resultToExport(rowIds?: number[], onProgress?: (info: { rowsExported: number; totalRows: number | null }) => void, useFullExport = true): Promise<{ columns: string[]; columnTypes: string[]; rows: CellValue[][] }> { + function applyGlobalDateTimeExportFormat(result: { columns: string[]; columnTypes: string[]; rows: CellValue[][] }, enabled: boolean) { + const pattern = enabled ? useSettingsStore().editorSettings.globalDateTimeExportFormat : ""; + return pattern ? { ...result, rows: formatTemporalRowsForExport(result.rows, result.columnTypes, pattern) } : result; + } + + async function resultToExport(rowIds?: number[], onProgress?: (info: { rowsExported: number; totalRows: number | null }) => void, useFullExport = true, formatDateTime = true): Promise<{ columns: string[]; columnTypes: string[]; rows: CellValue[][] }> { if (useFullExport && rowIds === undefined && fullExportResult && !hasCompleteLocalResult?.value) { const result = await fullExportResult(onProgress); - if (result) return { columns: result.columns, columnTypes: result.column_types ?? [], rows: result.rows }; + if (result) return applyGlobalDateTimeExportFormat({ columns: result.columns, columnTypes: result.column_types ?? [], rows: result.rows }, formatDateTime); } // The full result is already in memory — export the raw QueryResult (all // rows, all columns, committed values) so "export all data" matches the @@ -213,13 +219,16 @@ export function useDataGridExport(options: UseDataGridExportOptions) { // and reflects client-side filters/search and unsaved edits, which would // silently change what the export contains. if (useFullExport && rowIds === undefined && hasCompleteLocalResult?.value && completeLocalResult?.value) { - return { columns: completeLocalResult.value.columns, columnTypes: completeLocalResult.value.column_types ?? [], rows: completeLocalResult.value.rows }; + return applyGlobalDateTimeExportFormat({ columns: completeLocalResult.value.columns, columnTypes: completeLocalResult.value.column_types ?? [], rows: completeLocalResult.value.rows }, formatDateTime); } - return { - columns: columns.value, - columnTypes: (columnTypes.value ?? []).map((type) => type ?? ""), - rows: rowsToExport(rowIds).map((item) => item.data), - }; + return applyGlobalDateTimeExportFormat( + { + columns: columns.value, + columnTypes: (columnTypes.value ?? []).map((type) => type ?? ""), + rows: rowsToExport(rowIds).map((item) => item.data), + }, + formatDateTime, + ); } function currentXlsxSheetName(): string { @@ -1026,11 +1035,12 @@ export function useDataGridExport(options: UseDataGridExportOptions) { outputPath = path as string; } + const exportPattern = useSettingsStore().editorSettings.globalDateTimeExportFormat; const worksheets = sheets.map((sheet) => ({ sheetName: sheet.sheetName, columns: sheet.result.columns, columnTypes: sheet.result.column_types ?? [], - rows: sheet.result.rows, + rows: formatTemporalRowsForExport(sheet.result.rows, sheet.result.column_types ?? [], exportPattern), })); const sqlWorksheet = includeSqlSheet ? buildXlsxSqlWorksheet(sheets.map((sheet) => ({ resultName: sheet.sheetName, sql: sheet.sql || sheet.result.sourceStatement || "" }))) : undefined; await api.exportQueryResultsXlsx(outputPath, sqlWorksheet ? [...worksheets, sqlWorksheet] : worksheets); @@ -1110,6 +1120,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { skipCount: false, batchSize: exportBatchSize.value, rowLimit, + dateTimeFormat: editorSettings.globalDateTimeExportFormat || undefined, }, (progress) => { if (exportProgressState) { @@ -1156,7 +1167,8 @@ export function useDataGridExport(options: UseDataGridExportOptions) { } const exportId = uuid(); - const request = await queryResultExportRequest({ exportId, filePath: outputPath, format, includeSqlSheet }); + const baseRequest = await queryResultExportRequest({ exportId, filePath: outputPath, format, includeSqlSheet }); + const request = baseRequest ? { ...baseRequest, dateTimeFormat: useSettingsStore().editorSettings.globalDateTimeExportFormat || undefined } : undefined; if (!request) throw new Error("Unable to build query result export request"); if (exportProgressState) { @@ -1204,7 +1216,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { try { if (await exportFullTableDataViaBackend("sql", rowIds)) return; - const result = await resultToExport(rowIds); + const result = await resultToExport(rowIds, undefined, true, false); const exportData = sqlInsertExportData(result); const content = await formatSqlInsert({ databaseType: databaseType.value, @@ -1225,7 +1237,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { async function exportCurrentPageSql() { await runExclusiveExport(async () => { try { - const result = await resultToExport(undefined, undefined, false); + const result = await resultToExport(undefined, undefined, false, false); const exportData = sqlInsertExportData(result); const content = await formatSqlInsert({ databaseType: databaseType.value, diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts index b651c27d8..54054c397 100644 --- a/apps/desktop/src/i18n/locales/en.ts +++ b/apps/desktop/src/i18n/locales/en.ts @@ -3498,6 +3498,17 @@ export default { redisScanPageSizeDescription: "Keys requested per Redis SCAN page when browsing keys.", redisScanPageSizeOption: "{count} keys", exportBatchSize: "Export batch size", + dateTimeSection: "Date and time", + globalDateTimeDisplayFormat: "Global display format", + globalDateTimeDisplayFormatDescription: "Applies to temporal columns without a column-specific formatter in every database.", + globalDateTimeExportFormat: "Global export format", + globalDateTimeExportFormatDescription: "Formats temporal columns consistently in CSV, Excel, JSON, Markdown, and text exports.", + globalDateTimeImportFormat: "Global import format", + globalDateTimeImportFormatDescription: "Parses imported temporal values with this format. Leave empty to detect common formats automatically.", + dateTimeFormatRaw: "Keep database value", + dateTimeFormatAuto: "Auto detect", + dateTimeFormatSearchPlaceholder: "Select or enter a format", + dateTimeFormatEmpty: "Enter a custom datetime format", exportBatchSizeDescription: "Rows fetched per batch when exporting data (100-100000).", exportRowLimitEnabled: "Limit exported rows", exportRowLimitEnabledDescription: "When on, query result and table data exports stop at the row limit below.", diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts index 8553d29e8..97682b360 100644 --- a/apps/desktop/src/i18n/locales/es.ts +++ b/apps/desktop/src/i18n/locales/es.ts @@ -3474,6 +3474,17 @@ export default withEnglishFallback({ shortcutUppercaseSelection: "Convertir selección a mayúsculas", shortcutLowercaseSelection: "Convertir selección a minúsculas", shortcutExPasteSqlInCondition: "ExPaste: pegar como condición IN", + dateTimeSection: "Fecha y hora", + globalDateTimeDisplayFormat: "Formato de visualización global", + globalDateTimeDisplayFormatDescription: "Se aplica a todos los campos de fecha y hora que no tienen un formato de columna configurado individualmente en las bases de datos.", + globalDateTimeExportFormat: "Formato de exportación global", + globalDateTimeExportFormatDescription: "Formatea uniformemente los campos de fecha y hora al exportar a CSV, Excel, JSON, Markdown y texto.", + globalDateTimeImportFormat: "Formato de importación global", + globalDateTimeImportFormatDescription: "Analiza las fechas y horas en los archivos importados según este formato; si se deja vacío, se reconocen automáticamente los formatos comunes.", + dateTimeFormatRaw: "Mantener el valor original de la base de datos", + dateTimeFormatAuto: "Reconocimiento automático", + dateTimeFormatSearchPlaceholder: "Seleccionar o ingresar formato", + dateTimeFormatEmpty: "Ingresar formato de fecha y hora personalizado", }, driverStore: { progressJreExtract: "Extrayendo JRE...", diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts index e900486b8..071321044 100644 --- a/apps/desktop/src/i18n/locales/it.ts +++ b/apps/desktop/src/i18n/locales/it.ts @@ -3472,6 +3472,17 @@ export default withEnglishFallback({ shortcutUppercaseSelection: "Converti selezione in maiuscolo", shortcutLowercaseSelection: "Converti selezione in minuscolo", shortcutExPasteSqlInCondition: "ExPaste: incolla come condizione IN", + dateTimeSection: "Data e Ora", + globalDateTimeDisplayFormat: "Formato di visualizzazione globale", + globalDateTimeDisplayFormatDescription: "Applicato a tutti i campi data/ora nei database che non hanno una configurazione di formato colonna individuale.", + globalDateTimeExportFormat: "Formato di esportazione globale", + globalDateTimeExportFormatDescription: "Formatta uniformemente i campi data/ora durante l'esportazione in CSV, Excel, JSON, Markdown e testo.", + globalDateTimeImportFormat: "Formato di importazione globale", + globalDateTimeImportFormatDescription: "Analizza i campi data/ora nei file importati secondo questo formato; lascia vuoto per rilevare automaticamente i formati comuni.", + dateTimeFormatRaw: "Mantieni il valore originale del database", + dateTimeFormatAuto: "Riconoscimento automatico", + dateTimeFormatSearchPlaceholder: "Seleziona o inserisci il formato", + dateTimeFormatEmpty: "Inserisci un formato data/ora personalizzato", }, driverStore: { progressJreExtract: "Estrazione JRE...", diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts index 01efaddd1..69fa74887 100644 --- a/apps/desktop/src/i18n/locales/ja.ts +++ b/apps/desktop/src/i18n/locales/ja.ts @@ -3473,6 +3473,17 @@ export default withEnglishFallback({ shortcutUppercaseSelection: "選択範囲を大文字に変換", shortcutLowercaseSelection: "選択範囲を小文字に変換", shortcutExPasteSqlInCondition: "ExPaste: IN条件として貼り付け", + dateTimeSection: "日付と時刻", + globalDateTimeDisplayFormat: "グローバル表示形式", + globalDateTimeDisplayFormatDescription: "すべてのデータベースで列形式が個別に設定されていない日時フィールドに適用されます。", + globalDateTimeExportFormat: "グローバルエクスポート形式", + globalDateTimeExportFormatDescription: "CSV、Excel、JSON、Markdown、テキストをエクスポートする際に日時フィールドを統一フォーマットします。", + globalDateTimeImportFormat: "グローバルインポート形式", + globalDateTimeImportFormatDescription: "この形式でインポートファイル内の日時を解析します。空欄の場合は一般的な形式を自動認識します。", + dateTimeFormatRaw: "データベースの元の値を保持", + dateTimeFormatAuto: "自動認識", + dateTimeFormatSearchPlaceholder: "形式を選択または入力", + dateTimeFormatEmpty: "カスタム日時形式を入力", }, driverStore: { progressJreExtract: "JREを展開中...", diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts index 952478c2c..bd4932aa5 100644 --- a/apps/desktop/src/i18n/locales/pt-BR.ts +++ b/apps/desktop/src/i18n/locales/pt-BR.ts @@ -3474,6 +3474,17 @@ export default withEnglishFallback({ shortcutUppercaseSelection: "Converter seleção em maiúsculas", shortcutLowercaseSelection: "Converter seleção em minúsculas", shortcutExPasteSqlInCondition: "ExPaste: colar como condição IN", + dateTimeSection: "Data e Hora", + globalDateTimeDisplayFormat: "Formato de exibição global", + globalDateTimeDisplayFormatDescription: "Aplicado a todos os campos de data e hora em todos os bancos de dados que não tenham formato de coluna configurado individualmente.", + globalDateTimeExportFormat: "Formato de exportação global", + globalDateTimeExportFormatDescription: "Formata campos de data e hora de forma uniforme ao exportar para CSV, Excel, JSON, Markdown e texto.", + globalDateTimeImportFormat: "Formato de importação global", + globalDateTimeImportFormatDescription: "Analisa datas e horas em arquivos importados de acordo com este formato; deixe em branco para detectar automaticamente formatos comuns.", + dateTimeFormatRaw: "Manter valor original do banco de dados", + dateTimeFormatAuto: "Detecção automática", + dateTimeFormatSearchPlaceholder: "Selecione ou digite um formato", + dateTimeFormatEmpty: "Insira um formato de data e hora personalizado", }, driverStore: { progressJreExtract: "Extraindo JRE...", diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts index c39f3b7d3..22f2d5501 100644 --- a/apps/desktop/src/i18n/locales/zh-CN.ts +++ b/apps/desktop/src/i18n/locales/zh-CN.ts @@ -3488,6 +3488,17 @@ export default withEnglishFallback({ redisScanPageSizeDescription: "浏览 Redis Key 时每次 SCAN 请求的 Key 数量。", redisScanPageSizeOption: "{count} 个 Key", exportBatchSize: "导出批次大小", + dateTimeSection: "日期与时间", + globalDateTimeDisplayFormat: "全局显示格式", + globalDateTimeDisplayFormatDescription: "应用于所有数据库中未单独配置列格式的日期时间字段。", + globalDateTimeExportFormat: "全局导出格式", + globalDateTimeExportFormatDescription: "导出 CSV、Excel、JSON、Markdown 和文本时统一格式化日期时间字段。", + globalDateTimeImportFormat: "全局导入格式", + globalDateTimeImportFormatDescription: "按此格式解析导入文件中的日期时间；留空时自动识别常见格式。", + dateTimeFormatRaw: "保持数据库原始值", + dateTimeFormatAuto: "自动识别", + dateTimeFormatSearchPlaceholder: "选择或输入格式", + dateTimeFormatEmpty: "输入自定义日期时间格式", exportBatchSizeDescription: "导出数据时每批读取的行数（100-100000）。", exportRowLimitEnabled: "限制导出行数", exportRowLimitEnabledDescription: "开启时，查询结果和表数据导出最多为下方设置的行数。", diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts index fa5893aac..6471e3ca6 100644 --- a/apps/desktop/src/i18n/locales/zh-TW.ts +++ b/apps/desktop/src/i18n/locales/zh-TW.ts @@ -3319,6 +3319,17 @@ export default withEnglishFallback({ updateDownloadSourceAtomgit: "AtomGit", exportSection: "匯出", exportBatchSize: "匯出批次大小", + dateTimeSection: "日期與時間", + globalDateTimeDisplayFormat: "全域顯示格式", + globalDateTimeDisplayFormatDescription: "套用到所有資料庫中未單獨設定欄位格式的日期時間欄位。", + globalDateTimeExportFormat: "全域匯出格式", + globalDateTimeExportFormatDescription: "匯出 CSV、Excel、JSON、Markdown 和文字時統一格式化日期時間欄位。", + globalDateTimeImportFormat: "全域匯入格式", + globalDateTimeImportFormatDescription: "依此格式解析匯入檔案中的日期時間；留空時自動辨識常見格式。", + dateTimeFormatRaw: "保留資料庫原始值", + dateTimeFormatAuto: "自動辨識", + dateTimeFormatSearchPlaceholder: "選擇或輸入格式", + dateTimeFormatEmpty: "輸入自訂日期時間格式", exportBatchSizeDescription: "匯出資料時每批次擷取的列數（100-100000）。", exportRowLimitEnabled: "限制匯出行數", exportRowLimitEnabledDescription: "開啟後，查詢結果和資料表資料匯出將在下方設定的行數上限處停止。", diff --git a/apps/desktop/src/lib/__tests__/dataGrid/columnFormatter.spec.ts b/apps/desktop/src/lib/__tests__/dataGrid/columnFormatter.spec.ts new file mode 100644 index 000000000..232e42a3f --- /dev/null +++ b/apps/desktop/src/lib/__tests__/dataGrid/columnFormatter.spec.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import { normalizeSupportedDateTimePattern } from "@/lib/dataGrid/columnFormatter"; + +describe("normalizeSupportedDateTimePattern", () => { + it("accepts the format grammar shared by the frontend and backend", () => { + expect(normalizeSupportedDateTimePattern(" YYYY/M/D [at] HH:mm:ss.SSSZ ")).toBe("YYYY/M/D [at] HH:mm:ss.SSSZ"); + }); + + it("rejects unsupported or malformed Day.js tokens", () => { + expect(normalizeSupportedDateTimePattern("MM/DD/YYYY hh:mm A")).toBe(""); + expect(normalizeSupportedDateTimePattern("YYYY-MM-DD [at HH:mm:ss")).toBe(""); + expect(normalizeSupportedDateTimePattern("%Y-%m-%d")).toBe(""); + }); +}); diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts index 18d92a238..21d676813 100644 --- a/apps/desktop/src/lib/backend/tauri.ts +++ b/apps/desktop/src/lib/backend/tauri.ts @@ -2201,6 +2201,7 @@ export interface TableImportRequest { mode: TableImportMode; createTable?: boolean; batchSize: number; + dateTimeFormat?: string; } export interface TableImportSummary { @@ -2291,6 +2292,7 @@ export interface TableExportRequest { skipCount?: boolean; batchSize?: number; rowLimit?: number | null; + dateTimeFormat?: string; } export interface TableCsvExportOptions { @@ -2332,6 +2334,7 @@ export interface QueryResultExportRequest { keysetOptimizationEnabled: boolean; clientSessionId?: string; executionId?: string; + dateTimeFormat?: string; } export async function startTableExport(request: TableExportRequest, onProgress: (progress: TableExportProgress) => void): Promise { diff --git a/apps/desktop/src/lib/dataGrid/columnFormatter.ts b/apps/desktop/src/lib/dataGrid/columnFormatter.ts index 268346546..8b64911d4 100644 --- a/apps/desktop/src/lib/dataGrid/columnFormatter.ts +++ b/apps/desktop/src/lib/dataGrid/columnFormatter.ts @@ -10,9 +10,47 @@ dayjs.extend(timezone); export type DateTimeFormatterUnit = "seconds" | "milliseconds" | "auto"; const DEFAULT_DATETIME_PATTERN = "YYYY-MM-DD HH:mm:ss"; -export const DateTimePatterns = ["HH:mm:ss", "HH:mm:ss.SSS", "YYYY-MM-DD HH:mm:ss", "YYYY-MM-DD HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss", "YYYY/MM/DD HH:mm:ss.SSS", "YYYY-MM-DDTHH:mm:ssZ", "YYYY-MM-DDTHH:mm:ss.SSSZ", "YYYY/MM/DDTHH:mm:ssZ", "YYYY/MM/DDTHH:mm:ss.SSSZ"]; -const STRICT_LOCAL_DATETIME_INPUT_PATTERNS = ["YYYY-MM-DD", "YYYY/MM/DD", "YYYY-MM-DD HH:mm:ss", "YYYY-MM-DD HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss", "YYYY/MM/DD HH:mm:ss.SSS", "YYYY-MM-DDTHH:mm:ss", "YYYY-MM-DDTHH:mm:ss.SSS", "YYYY/MM/DDTHH:mm:ss", "YYYY/MM/DDTHH:mm:ss.SSS"]; -const ISO_OFFSET_DATETIME_PATTERN = /^(\d{4})([-/])(\d{2})\2(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d{1,3})?(Z|[+-]\d{2}:\d{2})$/; +export const DateTimePatterns = [ + "YYYY-MM-DD", + "YYYY/MM/DD", + "YYYY/M/D", + "HH:mm:ss", + "HH:mm:ss.SSS", + "YYYY-MM-DD HH:mm:ss", + "YYYY-MM-DD HH:mm:ss.SSS", + "YYYY/MM/DD HH:mm:ss", + "YYYY/MM/DD HH:mm:ss.SSS", + "YYYY/M/D HH:mm:ss", + "YYYY-MM-DDTHH:mm:ssZ", + "YYYY-MM-DDTHH:mm:ss.SSSZ", + "YYYY/MM/DDTHH:mm:ssZ", + "YYYY/MM/DDTHH:mm:ss.SSSZ", +]; +const SUPPORTED_DATE_TIME_PATTERN_TOKENS = ["YYYY", "SSS", "ZZ", "MM", "DD", "HH", "mm", "ss", "M", "D", "H", "m", "s", "Z"]; +const STRICT_LOCAL_DATETIME_INPUT_PATTERNS = [ + "YYYY-MM-DD", + "YYYY-M-D", + "YYYY/MM/DD", + "YYYY/M/D", + "YYYY-MM-DD HH:mm:ss", + "YYYY-M-D H:m:s", + "YYYY-MM-DD HH:mm:ss.SSS", + "YYYY-M-D H:m:s.SSS", + "YYYY/MM/DD HH:mm:ss", + "YYYY/M/D H:m:s", + "YYYY/MM/DD HH:mm:ss.SSS", + "YYYY/M/D H:m:s.SSS", + "YYYY-MM-DDTHH:mm:ss", + "YYYY-M-DTH:m:s", + "YYYY-MM-DDTHH:mm:ss.SSS", + "YYYY-M-DTH:m:s.SSS", + "YYYY/MM/DDTHH:mm:ss", + "YYYY/M/DTH:m:s", + "YYYY/MM/DDTHH:mm:ss.SSS", + "YYYY/M/DTH:m:s.SSS", +]; +const ISO_OFFSET_DATETIME_PATTERN = /^(\d{4})([-/])(\d{2})\2(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d{1,9})?(Z|[+-]\d{2}:\d{2})$/; +const FRACTIONAL_LOCAL_DATETIME_PATTERN = /^(\d{4})([-/])(\d{1,2})\2(\d{1,2})([ T])(\d{1,2}):(\d{1,2}):(\d{1,2})\.(\d{1,9})$/; export interface CustomColumnFormatterConfig { id: string; @@ -20,6 +58,33 @@ export interface CustomColumnFormatterConfig { template: string; } +export function normalizeSupportedDateTimePattern(value: string): string { + const pattern = value.trim(); + if (!pattern || pattern.length > 100 || pattern.includes("%")) return ""; + + let index = 0; + while (index < pattern.length) { + const remaining = pattern.slice(index); + if (remaining.startsWith("[")) { + const closeIndex = remaining.indexOf("]"); + if (closeIndex < 0) return ""; + index += closeIndex + 1; + continue; + } + + const token = SUPPORTED_DATE_TIME_PATTERN_TOKENS.find((candidate) => remaining.startsWith(candidate)); + if (token) { + index += token.length; + continue; + } + + if (/[A-Za-z]/.test(pattern[index])) return ""; + index += 1; + } + + return pattern; +} + export type ColumnFormatterConfig = { kind: "datetime"; unit: DateTimeFormatterUnit; pattern: string } | { kind: "json-path"; path: string } | { kind: "mask"; prefix: number; suffix: number } | { kind: "custom-template"; template: string } | { kind: "custom-ref"; formatterId: string }; export interface ColumnFormatterKeyParts { @@ -82,13 +147,69 @@ export function normalizeCustomColumnFormatter(value: unknown): CustomColumnForm }; } -export function resolveColumnFormatter(formatter: ColumnFormatterConfig | undefined, customFormatters: Record): ColumnFormatterConfig | undefined { - if (!formatter) return undefined; +export function normalizeGlobalDateTimePattern(value: unknown): string { + return typeof value === "string" ? value.trim().slice(0, 100) : ""; +} + +export function isTemporalColumnType(dataType: string | null | undefined): boolean { + const normalized = String(dataType || "") + .trim() + .toLowerCase() + .replace(/\s+/g, " "); + if (!normalized) return false; + const base = normalized.split(/[(:]/)[0]?.trim() ?? ""; + return ( + ["date", "date32", "daten", "time", "time64", "timen", "timetz", "datetime", "datetime2", "datetime4", "datetime64", "datetimen", "datetimeoffset", "datetimeoffsetn", "smalldatetime", "timestamp", "timestampdty", "timestamptz"].includes(base) || + base.startsWith("timestamp_") || + normalized.startsWith("timestamp with ") || + normalized.startsWith("timestamp without ") || + normalized.startsWith("time with ") || + normalized.startsWith("time without ") + ); +} + +export function resolveColumnFormatter(formatter: ColumnFormatterConfig | undefined, customFormatters: Record, globalDateTime?: { pattern?: string; columnType?: string | null }): ColumnFormatterConfig | undefined { + if (!formatter) { + const pattern = normalizeGlobalDateTimePattern(globalDateTime?.pattern); + return pattern && isTemporalColumnType(globalDateTime?.columnType) ? { kind: "datetime", unit: "auto", pattern } : undefined; + } if (formatter.kind !== "custom-ref") return formatter; const customFormatter = customFormatters[formatter.formatterId]; return customFormatter ? { kind: "custom-template", template: customFormatter.template } : undefined; } +export function formatTemporalRowsForExport(rows: readonly (readonly T[])[], columnTypes: readonly (string | null | undefined)[], pattern: string): T[][] { + const normalizedPattern = normalizeGlobalDateTimePattern(pattern); + if (!normalizedPattern) return rows.map((row) => [...row]); + return rows.map((row) => + row.map((value, index) => { + if (!isTemporalColumnType(columnTypes[index])) return value; + return formatTemporalValueForExport(value, normalizedPattern) as T; + }), + ); +} + +function formatTemporalValueForExport(value: CellValue, pattern: string): string { + if (typeof value === "string") { + const match = value.match(ISO_OFFSET_DATETIME_PATTERN); + if (match) { + const [, year, separator, month, day, hour, minute, second, fraction = "", zone] = match; + if (!isValidDateTimeParts(year, month, day, hour, minute, second) || !isValidOffset(zone)) { + return displayCellValue(value); + } + const normalizedFraction = fraction ? `.${fraction.slice(1, 4).padEnd(3, "0")}` : ""; + const localValue = `${year}${separator}${month}${separator}${day}T${hour}:${minute}:${second}${normalizedFraction}`; + const inputPattern = `${separator === "-" ? "YYYY-MM-DD" : "YYYY/MM/DD"}THH:mm:ss${fraction ? ".SSS" : ""}`; + const parsed = dayjs(localValue, inputPattern, true); + if (parsed.isValid()) { + const offsetMinutes = zone === "Z" ? 0 : (zone.startsWith("-") ? -1 : 1) * (Number(zone.slice(1, 3)) * 60 + Number(zone.slice(4, 6))); + return parsed.utcOffset(offsetMinutes, true).format(pattern); + } + } + } + return applyColumnFormatter(value, { kind: "datetime", unit: "auto", pattern }); +} + export function applyColumnFormatter(value: CellValue, formatter: ColumnFormatterConfig | undefined): string { if (!formatter) return displayCellValue(value); @@ -142,6 +263,16 @@ function parseStrictDateTimeString(value: string): Dayjs | undefined { const parsedIsoOffset = parseIsoOffsetDateTimeString(value); if (parsedIsoOffset) return parsedIsoOffset; + const fractionalLocalMatch = value.match(FRACTIONAL_LOCAL_DATETIME_PATTERN); + if (fractionalLocalMatch) { + const [, yearText, , monthText, dayText, , hourText, minuteText, secondText, fractionText] = fractionalLocalMatch; + if (isValidDateTimeParts(yearText, monthText, dayText, hourText, minuteText, secondText)) { + const normalized = `${yearText}-${monthText.padStart(2, "0")}-${dayText.padStart(2, "0")}T${hourText.padStart(2, "0")}:${minuteText.padStart(2, "0")}:${secondText.padStart(2, "0")}.${fractionText.slice(0, 3).padEnd(3, "0")}`; + const parsed = dayjs(normalized, "YYYY-MM-DDTHH:mm:ss.SSS", true); + if (parsed.isValid()) return parsed; + } + } + for (const pattern of STRICT_LOCAL_DATETIME_INPUT_PATTERNS) { // Day.js non-strict parsing normalizes overflow dates such as 2022-01-33. // Keep cell text strict so invalid values fall back unchanged. @@ -159,7 +290,8 @@ function parseIsoOffsetDateTimeString(value: string): Dayjs | undefined { if (!isValidDateTimeParts(yearText, monthText, dayText, hourText, minuteText, secondText)) return undefined; if (!isValidOffset(zoneText)) return undefined; - const normalized = `${yearText}-${monthText}-${dayText}T${hourText}:${minuteText}:${secondText}${fractionText}${zoneText}`; + const normalizedFraction = fractionText ? `.${fractionText.slice(1, 4).padEnd(3, "0")}` : ""; + const normalized = `${yearText}-${monthText}-${dayText}T${hourText}:${minuteText}:${secondText}${normalizedFraction}${zoneText}`; const parsed = dayjs(normalized); return parsed.isValid() ? parsed : undefined; } diff --git a/apps/desktop/src/lib/settings/editorSettingsDraft.ts b/apps/desktop/src/lib/settings/editorSettingsDraft.ts index 8de49aecf..98273dd82 100644 --- a/apps/desktop/src/lib/settings/editorSettingsDraft.ts +++ b/apps/desktop/src/lib/settings/editorSettingsDraft.ts @@ -47,6 +47,9 @@ export const EDITOR_SETTINGS_DRAFT_KEYS = [ "exportRowLimitEnabled", "exportRowLimit", "queryExportKeysetOptimizationEnabled", + "globalDateTimeDisplayFormat", + "globalDateTimeExportFormat", + "globalDateTimeImportFormat", "updateDownloadSource", "toolbarItems", "snippets", diff --git a/apps/desktop/src/stores/settingsStore.ts b/apps/desktop/src/stores/settingsStore.ts index ceefcde61..8afa60521 100644 --- a/apps/desktop/src/stores/settingsStore.ts +++ b/apps/desktop/src/stores/settingsStore.ts @@ -1,7 +1,7 @@ import { defineStore } from "pinia"; import { ref } from "vue"; import * as api from "@/lib/backend/api"; -import { normalizeColumnFormatter, normalizeCustomColumnFormatter, type ColumnFormatterConfig, type CustomColumnFormatterConfig } from "@/lib/dataGrid/columnFormatter"; +import { normalizeColumnFormatter, normalizeCustomColumnFormatter, normalizeGlobalDateTimePattern, type ColumnFormatterConfig, type CustomColumnFormatterConfig } from "@/lib/dataGrid/columnFormatter"; import { normalizeShortcutSettings, type ShortcutSettings } from "@/lib/editor/shortcutRegistry"; import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize"; import { normalizeSidebarHiddenTablePrefixes } from "@/lib/sidebar/sidebarTableNameDisplay"; @@ -421,6 +421,9 @@ export interface EditorSettings { sidebarAllowHorizontalScroll: boolean; columnFormatters: Record; customColumnFormatters: Record; + globalDateTimeDisplayFormat: string; + globalDateTimeExportFormat: string; + globalDateTimeImportFormat: string; snippets: SqlSnippet[]; tableColumnTemplateFields: string[]; exportBatchSize: number; @@ -559,6 +562,9 @@ export const DEFAULT_EDITOR_SETTINGS: EditorSettings = { sidebarAllowHorizontalScroll: false, columnFormatters: {}, customColumnFormatters: {}, + globalDateTimeDisplayFormat: "", + globalDateTimeExportFormat: "", + globalDateTimeImportFormat: "", snippets: DEFAULT_SQL_SNIPPETS, tableColumnTemplateFields: [...DEFAULT_TABLE_COLUMN_TEMPLATE_FIELDS], exportBatchSize: 2000, @@ -797,6 +803,9 @@ export function normalizeEditorSettings(settings: Partial, exist sidebarAllowHorizontalScroll: settings.sidebarAllowHorizontalScroll ?? DEFAULT_EDITOR_SETTINGS.sidebarAllowHorizontalScroll, columnFormatters: normalizeColumnFormatters(settings.columnFormatters), customColumnFormatters: normalizeCustomColumnFormatters(settings.customColumnFormatters), + globalDateTimeDisplayFormat: normalizeGlobalDateTimePattern(settings.globalDateTimeDisplayFormat), + globalDateTimeExportFormat: normalizeGlobalDateTimePattern(settings.globalDateTimeExportFormat), + globalDateTimeImportFormat: normalizeGlobalDateTimePattern(settings.globalDateTimeImportFormat), snippets: normalizeSqlSnippets(settings.snippets, existing?.snippets), tableColumnTemplateFields: normalizeTableColumnTemplateFields(settings.tableColumnTemplateFields), exportBatchSize: typeof settings.exportBatchSize === "number" && settings.exportBatchSize >= 100 && settings.exportBatchSize <= 100000 ? Math.round(settings.exportBatchSize) : DEFAULT_EDITOR_SETTINGS.exportBatchSize, @@ -1047,6 +1056,9 @@ export const useSettingsStore = defineStore("settings", () => { if (partial.sidebarAllowHorizontalScroll !== undefined) editorSettings.value.sidebarAllowHorizontalScroll = partial.sidebarAllowHorizontalScroll; if (partial.columnFormatters !== undefined) editorSettings.value.columnFormatters = partial.columnFormatters; if (partial.customColumnFormatters !== undefined) editorSettings.value.customColumnFormatters = partial.customColumnFormatters; + if (partial.globalDateTimeDisplayFormat !== undefined) editorSettings.value.globalDateTimeDisplayFormat = normalizeGlobalDateTimePattern(partial.globalDateTimeDisplayFormat); + if (partial.globalDateTimeExportFormat !== undefined) editorSettings.value.globalDateTimeExportFormat = normalizeGlobalDateTimePattern(partial.globalDateTimeExportFormat); + if (partial.globalDateTimeImportFormat !== undefined) editorSettings.value.globalDateTimeImportFormat = normalizeGlobalDateTimePattern(partial.globalDateTimeImportFormat); if (partial.snippets !== undefined) editorSettings.value.snippets = normalizeSqlSnippets(partial.snippets); if (partial.tableColumnTemplateFields !== undefined) editorSettings.value.tableColumnTemplateFields = normalizeTableColumnTemplateFields(partial.tableColumnTemplateFields); if (partial.exportBatchSize !== undefined) editorSettings.value.exportBatchSize = Math.min(100000, Math.max(100, Math.round(partial.exportBatchSize))); diff --git a/crates/dbx-core/src/db/sqlserver.rs b/crates/dbx-core/src/db/sqlserver.rs index 823f69e36..bdcf91745 100644 --- a/crates/dbx-core/src/db/sqlserver.rs +++ b/crates/dbx-core/src/db/sqlserver.rs @@ -256,7 +256,7 @@ pub struct SqlServerStreamExportSummary { } pub enum SqlServerStreamItem<'a> { - Columns(&'a [String]), + Columns { columns: &'a [String], column_types: &'a [String] }, Row(&'a [serde_json::Value]), } @@ -638,6 +638,7 @@ pub async fn stream_first_result_set( let mut stream = sqlserver_driver_result(client.query(query_sql.as_str(), &[])).await?; let mut active_result_index: Option = None; let mut columns: Vec = Vec::new(); + let mut column_types: Vec = Vec::new(); let mut columns_emitted = false; let mut rows_exported = 0_u64; @@ -663,7 +664,8 @@ pub async fn stream_first_result_set( if active_result_index.is_none() { active_result_index = Some(metadata.result_index()); columns = columns_from_metadata(&metadata); - on_item(SqlServerStreamItem::Columns(&columns))?; + column_types = column_types_from_metadata(&metadata); + on_item(SqlServerStreamItem::Columns { columns: &columns, column_types: &column_types })?; columns_emitted = true; } } @@ -671,7 +673,8 @@ pub async fn stream_first_result_set( if active_result_index.is_none() { active_result_index = Some(row.result_index()); columns = row.columns().iter().map(|c| c.name().to_string()).collect(); - on_item(SqlServerStreamItem::Columns(&columns))?; + column_types = row.columns().iter().map(sqlserver_column_type_name).collect(); + on_item(SqlServerStreamItem::Columns { columns: &columns, column_types: &column_types })?; columns_emitted = true; } if Some(row.result_index()) != active_result_index { @@ -688,7 +691,7 @@ pub async fn stream_first_result_set( } if !columns_emitted { - on_item(SqlServerStreamItem::Columns(&columns))?; + on_item(SqlServerStreamItem::Columns { columns: &columns, column_types: &column_types })?; } Ok(SqlServerStreamExportSummary { columns, rows_exported }) } diff --git a/crates/dbx-core/src/lib.rs b/crates/dbx-core/src/lib.rs index 3a42d3655..98f075233 100644 --- a/crates/dbx-core/src/lib.rs +++ b/crates/dbx-core/src/lib.rs @@ -61,6 +61,7 @@ pub mod table_export; pub mod table_import; pub mod table_structure_sql; pub mod task_supervisor; +pub mod temporal_format; pub mod text_export; pub mod token_usage; pub mod transfer; diff --git a/crates/dbx-core/src/query_result_export.rs b/crates/dbx-core/src/query_result_export.rs index cd168acf2..cc75be08e 100644 --- a/crates/dbx-core/src/query_result_export.rs +++ b/crates/dbx-core/src/query_result_export.rs @@ -81,6 +81,8 @@ pub struct QueryResultExportRequest { pub client_session_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub execution_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub date_time_format: Option, } fn split_excel_cell_text(value: &str) -> Vec { @@ -563,6 +565,11 @@ async fn export_query_result_core_inner( result.rows.truncate(this_page); } let row_count = result.rows.len(); + let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types( + &result.rows, + &column_types, + request.date_time_format.as_deref(), + ); if format == "csv" || format == "txt" { if let Some(file) = text_file.as_mut() { @@ -570,12 +577,12 @@ async fn export_query_result_core_inner( let header = format_text_export_header(&format, &columns); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; if row_count > 0 { - let rows = format_text_export_rows(&format, &result.rows); + let rows = format_text_export_rows(&format, &formatted_rows); write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; } wrote_text_header = true; } else if row_count > 0 { - let rows = format_text_export_rows(&format, &result.rows); + let rows = format_text_export_rows(&format, &formatted_rows); write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; } } @@ -591,7 +598,7 @@ async fn export_query_result_core_inner( )?); } if let Some(writer) = xlsx.as_mut() { - for row in &result.rows { + for row in &formatted_rows { writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } } @@ -708,6 +715,7 @@ async fn try_export_postgres_query_result_stream( if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit }; let progress_row_interval = request.page_size.max(1) as u64; let mut columns: Vec = Vec::new(); + let mut temporal_column_types: Vec = Vec::new(); let mut rows_exported = 0_u64; let mut last_progress_rows = 0_u64; let mut last_progress_at = Instant::now(); @@ -735,6 +743,7 @@ async fn try_export_postgres_query_result_stream( match item { crate::db::postgres::PostgresQueryStreamItem::Columns { columns: stream_columns, column_types } => { columns = stream_columns; + temporal_column_types = column_types.clone(); if let Some(file) = text_file.as_mut() { let header = format_text_export_header(format, &columns); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; @@ -753,18 +762,23 @@ async fn try_export_postgres_query_result_stream( if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS { return Err(XLSX_ROW_LIMIT_ERROR.to_string()); } + let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( + &row, + &temporal_column_types, + request.date_time_format.as_deref(), + ); if let Some(file) = text_file.as_mut() { - let rows = format_text_export_rows(format, std::slice::from_ref(&row)); + let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; } else if let Some(writer) = xlsx.as_mut() { - writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } else { let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; xlsx = Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); if let Some(writer) = xlsx.as_mut() { - writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } } rows_exported += 1; @@ -870,6 +884,7 @@ async fn try_export_mysql_query_result_stream( if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit }; let progress_row_interval = request.page_size.max(1) as u64; let mut columns: Vec = Vec::new(); + let mut temporal_column_types: Vec = Vec::new(); let mut rows_exported = 0_u64; let mut last_progress_rows = 0_u64; let mut last_progress_at = Instant::now(); @@ -946,6 +961,7 @@ async fn try_export_mysql_query_result_stream( match item { crate::db::mysql::MySqlQueryStreamItem::Columns { columns: stream_columns, column_types } => { columns = stream_columns; + temporal_column_types = column_types.clone(); if let Some(file) = text_file.as_mut() { let header = format_text_export_header(format, &columns); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; @@ -964,18 +980,23 @@ async fn try_export_mysql_query_result_stream( if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS { return Err(XLSX_ROW_LIMIT_ERROR.to_string()); } + let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( + &row, + &temporal_column_types, + request.date_time_format.as_deref(), + ); if let Some(file) = text_file.as_mut() { - let rows = format_text_export_rows(format, std::slice::from_ref(&row)); + let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; } else if let Some(writer) = xlsx.as_mut() { - writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } else { let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; xlsx = Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); if let Some(writer) = xlsx.as_mut() { - writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } } rows_exported += 1; @@ -1115,6 +1136,7 @@ async fn try_export_clickhouse_query_result_stream( if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit }; let progress_row_interval = request.page_size.max(1) as u64; let mut columns: Vec = Vec::new(); + let mut temporal_column_types: Vec = Vec::new(); let mut rows_exported = 0_u64; let mut last_progress_rows = 0_u64; let mut last_progress_at = Instant::now(); @@ -1142,6 +1164,7 @@ async fn try_export_clickhouse_query_result_stream( column_types, } => { columns = stream_columns; + temporal_column_types = column_types.clone(); if let Some(file) = text_file.as_mut() { let header = format_text_export_header(format, &columns); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; @@ -1160,18 +1183,23 @@ async fn try_export_clickhouse_query_result_stream( if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS { return Err(XLSX_ROW_LIMIT_ERROR.to_string()); } + let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( + &row, + &temporal_column_types, + request.date_time_format.as_deref(), + ); if let Some(file) = text_file.as_mut() { - let rows = format_text_export_rows(format, std::slice::from_ref(&row)); + let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; } else if let Some(writer) = xlsx.as_mut() { - writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } else { let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; xlsx = Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); if let Some(writer) = xlsx.as_mut() { - writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } } rows_exported += 1; @@ -1267,6 +1295,7 @@ async fn try_export_sqlserver_query_result_stream( let stream_row_limit = if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit }; let mut columns: Vec = Vec::new(); + let mut temporal_column_types: Vec = Vec::new(); let mut rows_exported = 0_u64; let mut last_progress_rows = 0_u64; let mut last_progress_at = Instant::now(); @@ -1299,8 +1328,9 @@ async fn try_export_sqlserver_query_result_stream( cancel_token.clone(), |item| { match item { - crate::db::sqlserver::SqlServerStreamItem::Columns(stream_columns) => { + crate::db::sqlserver::SqlServerStreamItem::Columns { columns: stream_columns, column_types } => { columns = stream_columns.to_vec(); + temporal_column_types = column_types.to_vec(); if let Some(file) = text_file.as_mut() { let header = format_text_export_header(format, &columns); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; @@ -1315,18 +1345,23 @@ async fn try_export_sqlserver_query_result_stream( if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS { return Err(XLSX_ROW_LIMIT_ERROR.to_string()); } + let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( + row, + &temporal_column_types, + request.date_time_format.as_deref(), + ); if let Some(file) = text_file.as_mut() { - let rows = format_text_export_rows(format, &[row.to_vec()]); + let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; } else if let Some(writer) = xlsx.as_mut() { - writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } else { let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; xlsx = Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); if let Some(writer) = xlsx.as_mut() { - writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } } rows_exported += 1; @@ -1394,6 +1429,7 @@ mod tests { keyset_optimization_enabled: true, client_session_id: None, execution_id: None, + date_time_format: None, } } diff --git a/crates/dbx-core/src/table_export.rs b/crates/dbx-core/src/table_export.rs index 9a399e650..a00064601 100644 --- a/crates/dbx-core/src/table_export.rs +++ b/crates/dbx-core/src/table_export.rs @@ -56,6 +56,8 @@ pub struct TableExportRequest { pub batch_size: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub row_limit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub date_time_format: Option, } #[derive(Debug, Clone, Serialize)] @@ -516,7 +518,12 @@ async fn try_export_native_table_stream( &cancelled, cancel_token.clone(), |row| { - let row_csv = format_csv_rows(&[row.to_vec()]); + let formatted = crate::temporal_format::format_temporal_export_row( + row, + column_types, + request.date_time_format.as_deref(), + ); + let row_csv = format_csv_rows(&[formatted]); write!(file, "\n{row_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?; rows_exported += 1; if rows_exported % progress_interval == 0 { @@ -555,7 +562,12 @@ async fn try_export_native_table_stream( &cancelled, cancel_token.clone(), |row| { - let row_tsv = format_tsv_rows(&[row.to_vec()]); + let formatted = crate::temporal_format::format_temporal_export_row( + row, + column_types, + request.date_time_format.as_deref(), + ); + let row_tsv = format_tsv_rows(&[formatted]); write!(file, "\n{row_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?; rows_exported += 1; if rows_exported % progress_interval == 0 { @@ -578,14 +590,14 @@ async fn try_export_native_table_stream( result } "xlsx" => { - let column_types = export_column_types(request); + let xlsx_column_types = export_column_types(request); let xlsx_file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; let mut writer = start_streaming_xlsx_workbook( BufWriter::new(xlsx_file), Some(&request.table_name), col_names, - &column_types, + &xlsx_column_types, )?; let result = stream_native_table_rows( state, @@ -596,7 +608,12 @@ async fn try_export_native_table_stream( &cancelled, cancel_token.clone(), |row| { - writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + let formatted = crate::temporal_format::format_temporal_export_row( + row, + column_types, + request.date_time_format.as_deref(), + ); + writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; rows_exported += 1; if rows_exported % progress_interval == 0 { on_progress(TableExportProgress { @@ -645,7 +662,12 @@ async fn try_export_native_table_stream( if !is_first_row { file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?; } - write_json_row_object(&mut file, col_names, row)?; + let formatted = crate::temporal_format::format_temporal_export_row( + row, + column_types, + request.date_time_format.as_deref(), + ); + write_json_row_object(&mut file, col_names, &formatted)?; is_first_row = false; rows_exported += 1; if rows_exported % progress_interval == 0 { @@ -684,7 +706,12 @@ async fn try_export_native_table_stream( &cancelled, cancel_token.clone(), |row| { - let rows_markdown = format_markdown_rows(&[row.to_vec()]); + let formatted = crate::temporal_format::format_temporal_export_row( + row, + column_types, + request.date_time_format.as_deref(), + ); + let rows_markdown = format_markdown_rows(&[formatted]); if !rows_markdown.is_empty() { if wrote_rows { file.write_all(b"\n").map_err(|e| format!("Failed to write Markdown: {e}"))?; @@ -1016,15 +1043,20 @@ pub async fn export_table_data_core( if row_count == 0 { break; } + let formatted_rows = crate::temporal_format::format_temporal_export_rows( + &result.rows, + &column_types, + request.date_time_format.as_deref(), + ); if is_first_batch { // First batch: write header + rows via format_csv - let csv_content = format_csv(&col_names, &result.rows); + let csv_content = format_csv(&col_names, &formatted_rows); file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?; is_first_batch = false; } else { // Subsequent batches: write rows only (prepend newline for separation) - let rows_csv = format_csv_rows(&result.rows); + let rows_csv = format_csv_rows(&formatted_rows); if !rows_csv.is_empty() { write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?; } @@ -1098,13 +1130,18 @@ pub async fn export_table_data_core( if row_count == 0 { break; } + let formatted_rows = crate::temporal_format::format_temporal_export_rows( + &result.rows, + &column_types, + request.date_time_format.as_deref(), + ); if is_first_batch { - let rows_tsv = format_tsv_rows(&result.rows); + let rows_tsv = format_tsv_rows(&formatted_rows); write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?; is_first_batch = false; } else { - let rows_tsv = format_tsv_rows(&result.rows); + let rows_tsv = format_tsv_rows(&formatted_rows); if !rows_tsv.is_empty() { write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?; } @@ -1135,7 +1172,7 @@ pub async fn export_table_data_core( } } "xlsx" => { - let column_types = export_column_types(request); + let xlsx_column_types = export_column_types(request); // Create a dedicated file handle for the streaming XLSX writer // instead of cloning the outer BufWriter's handle. This avoids // sharing a file descriptor between two independent buffers. @@ -1145,7 +1182,7 @@ pub async fn export_table_data_core( BufWriter::new(xlsx_file), Some(&request.table_name), &col_names, - &column_types, + &xlsx_column_types, )?; loop { @@ -1188,7 +1225,12 @@ pub async fn export_table_data_core( } for row in &result.rows { - writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + let formatted = crate::temporal_format::format_temporal_export_row( + row, + &column_types, + request.date_time_format.as_deref(), + ); + writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } rows_exported += row_count as u64; @@ -1278,7 +1320,12 @@ pub async fn export_table_data_core( if !is_first_row { file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?; } - write_json_row_object(&mut file, &col_names, row)?; + let formatted = crate::temporal_format::format_temporal_export_row( + row, + &column_types, + request.date_time_format.as_deref(), + ); + write_json_row_object(&mut file, &col_names, &formatted)?; is_first_row = false; } @@ -1350,7 +1397,12 @@ pub async fn export_table_data_core( break; } - let rows_markdown = format_markdown_rows(&result.rows); + let formatted_rows = crate::temporal_format::format_temporal_export_rows( + &result.rows, + &column_types, + request.date_time_format.as_deref(), + ); + let rows_markdown = format_markdown_rows(&formatted_rows); if !rows_markdown.is_empty() { if wrote_rows { file.write_all(b"\n").map_err(|e| format!("Failed to write Markdown: {e}"))?; @@ -1622,6 +1674,7 @@ mod tests { skip_count: false, batch_size: Some(500), row_limit: Some(1000), + date_time_format: None, }; let sql = table_cursor_sql( @@ -1669,6 +1722,7 @@ mod tests { skip_count: false, batch_size: Some(100), row_limit: None, + date_time_format: None, }; let sql = table_cursor_sql(&request, &DatabaseType::Oracle, &columns, &primary_keys); assert_eq!(sql, "SELECT \"ID\", \"NAME\" FROM \"APP\".\"USERS\""); diff --git a/crates/dbx-core/src/table_import.rs b/crates/dbx-core/src/table_import.rs index 0e5fe65a2..7b940ea96 100644 --- a/crates/dbx-core/src/table_import.rs +++ b/crates/dbx-core/src/table_import.rs @@ -200,6 +200,8 @@ pub struct TableImportRequest { #[serde(default)] pub create_table: bool, pub batch_size: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub date_time_format: Option, } #[derive(Debug, Clone, Serialize)] @@ -1334,6 +1336,29 @@ pub fn build_import_insert_batch_from_rows( table: &str, schema: &str, db_type: &DatabaseType, +) -> Result, String> { + build_import_insert_batch_from_rows_with_format( + rows, + columns, + mappings, + target_column_types, + table, + schema, + db_type, + None, + ) +} + +#[allow(clippy::too_many_arguments)] +fn build_import_insert_batch_from_rows_with_format( + rows: &[Vec], + columns: &[String], + mappings: &[TableImportColumnMapping], + target_column_types: &[(String, String)], + table: &str, + schema: &str, + db_type: &DatabaseType, + date_time_format: Option<&str>, ) -> Result, String> { if rows.is_empty() { return Ok(None); @@ -1365,7 +1390,16 @@ pub fn build_import_insert_batch_from_rows( .map(|row| { mapped .iter() - .map(|(source_index, _)| row.get(*source_index).cloned().unwrap_or(serde_json::Value::Null)) + .enumerate() + .map(|(target_index, (source_index, _))| { + let value = row.get(*source_index).cloned().unwrap_or(serde_json::Value::Null); + normalize_import_temporal_value( + &value, + column_types.get(target_index).and_then(|data_type| data_type.as_deref()), + db_type, + date_time_format, + ) + }) .collect::>() }) .collect::>(); @@ -1377,6 +1411,21 @@ fn supports_multi_row_insert_values(db_type: &DatabaseType) -> bool { !matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Iris) } +fn normalize_import_temporal_value( + value: &serde_json::Value, + data_type: Option<&str>, + db_type: &DatabaseType, + date_time_format: Option<&str>, +) -> serde_json::Value { + let oracle_date_time = matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle) + && data_type.is_some_and(|data_type| data_type.trim().eq_ignore_ascii_case("date")); + crate::temporal_format::normalize_temporal_import_value( + value, + if oracle_date_time { Some("datetime") } else { data_type }, + date_time_format, + ) +} + pub fn build_import_insert_batches( data: &ParsedImportFile, mappings: &[TableImportColumnMapping], @@ -1385,6 +1434,29 @@ pub fn build_import_insert_batches( schema: &str, db_type: &DatabaseType, batch_size: usize, +) -> Result, String> { + build_import_insert_batches_with_format( + data, + mappings, + target_column_types, + table, + schema, + db_type, + batch_size, + None, + ) +} + +#[allow(clippy::too_many_arguments)] +fn build_import_insert_batches_with_format( + data: &ParsedImportFile, + mappings: &[TableImportColumnMapping], + target_column_types: &[(String, String)], + table: &str, + schema: &str, + db_type: &DatabaseType, + batch_size: usize, + date_time_format: Option<&str>, ) -> Result, String> { if *db_type == DatabaseType::CloudflareD1 { return crate::db::cloudflare_d1::build_import_insert_batches( @@ -1419,7 +1491,16 @@ pub fn build_import_insert_batches( .map(|row| { mapped .iter() - .map(|(source_index, _)| row.get(*source_index).cloned().unwrap_or(serde_json::Value::Null)) + .enumerate() + .map(|(target_index, (source_index, _))| { + let value = row.get(*source_index).cloned().unwrap_or(serde_json::Value::Null); + normalize_import_temporal_value( + &value, + column_types.get(target_index).and_then(|data_type| data_type.as_deref()), + db_type, + date_time_format, + ) + }) .collect::>() }) .collect::>(); @@ -2036,7 +2117,7 @@ where pending_rows.push(delimited_record_to_row(&record, columns.len(), config)); if pending_rows.len() >= effective_batch_size { - let batch = match build_import_insert_batch_from_rows( + let batch = match build_import_insert_batch_from_rows_with_format( &pending_rows, &columns, &request.mappings, @@ -2044,6 +2125,7 @@ where &request.table, &request.schema, db_type, + request.date_time_format.as_deref(), ) { Ok(Some(batch)) => batch, Ok(None) => { @@ -2086,7 +2168,7 @@ where }); return Err("Import cancelled".to_string()); } - let batch = match build_import_insert_batch_from_rows( + let batch = match build_import_insert_batch_from_rows_with_format( &pending_rows, &columns, &request.mappings, @@ -2094,6 +2176,7 @@ where &request.table, &request.schema, db_type, + request.date_time_format.as_deref(), ) { Ok(Some(batch)) => batch, Ok(None) => ImportSqlBatch { sql: String::new(), row_count: 0 }, @@ -2163,7 +2246,7 @@ where target_column_types = created_column_types.clone().unwrap_or_default(); } - let batches = match build_import_insert_batches( + let batches = match build_import_insert_batches_with_format( &parsed, &request.mappings, &target_column_types, @@ -2171,6 +2254,7 @@ where &request.schema, db_type, batch_size, + request.date_time_format.as_deref(), ) { Ok(batches) => batches, Err(error) => { @@ -3115,6 +3199,85 @@ mod tests { }]); } + #[test] + fn import_insert_batches_normalize_oracle_unpadded_slash_dates() { + let mappings = vec![TableImportColumnMapping { + source_column: "created_at".to_string(), + target_column: "created_at".to_string(), + target_data_type: None, + }]; + let data = ParsedImportFile { + columns: vec!["created_at".to_string()], + rows: vec![vec![serde_json::json!("2024/2/25 13:02:15")]], + total_rows: 1, + effective_encoding: None, + }; + + let batches = build_import_insert_batches( + &data, + &mappings, + &[("created_at".to_string(), "DATE".to_string())], + "events", + "APP", + &DatabaseType::Oracle, + 500, + ) + .unwrap(); + + assert_eq!( + batches[0].sql, + "INSERT INTO \"APP\".\"events\" (\"created_at\") VALUES\n(TO_DATE('2024-02-25 13:02:15', 'YYYY-MM-DD HH24:MI:SS'))" + ); + } + + #[test] + fn import_insert_batch_normalizes_oracle_date_and_timestamp_columns() { + let mappings = vec![ + TableImportColumnMapping { + source_column: "event_id".to_string(), + target_column: "EVENT_ID".to_string(), + target_data_type: None, + }, + TableImportColumnMapping { + source_column: "created_at".to_string(), + target_column: "CREATED_AT".to_string(), + target_data_type: None, + }, + TableImportColumnMapping { + source_column: "updated_at".to_string(), + target_column: "UPDATED_AT".to_string(), + target_data_type: None, + }, + ]; + let rows = vec![vec![ + serde_json::json!(1), + serde_json::json!("2024/2/25 13:02:15"), + serde_json::json!("2024/2/25 14:03:16"), + ]]; + + let batch = build_import_insert_batch_from_rows_with_format( + &rows, + &["event_id".to_string(), "created_at".to_string(), "updated_at".to_string()], + &mappings, + &[ + ("EVENT_ID".to_string(), "NUMBER".to_string()), + ("CREATED_AT".to_string(), "DATE".to_string()), + ("UPDATED_AT".to_string(), "TIMESTAMP(6)".to_string()), + ], + "EVENTS", + "APP", + &DatabaseType::Oracle, + Some("YYYY/M/D HH:mm:ss"), + ) + .unwrap() + .unwrap(); + + assert_eq!( + batch.sql, + "INSERT INTO \"APP\".\"EVENTS\" (\"EVENT_ID\", \"CREATED_AT\", \"UPDATED_AT\") VALUES\n(1, TO_DATE('2024-02-25 13:02:15', 'YYYY-MM-DD HH24:MI:SS'), TO_TIMESTAMP('2024-02-25 14:03:16', 'YYYY-MM-DD HH24:MI:SS'))" + ); + } + #[test] fn import_insert_batches_preserve_sqlserver_unicode_text() { let mappings = vec![TableImportColumnMapping { diff --git a/crates/dbx-core/src/temporal_format.rs b/crates/dbx-core/src/temporal_format.rs new file mode 100644 index 000000000..3d16264c7 --- /dev/null +++ b/crates/dbx-core/src/temporal_format.rs @@ -0,0 +1,321 @@ +use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; +use serde_json::Value; +use std::fmt::Write as _; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TemporalKind { + Date, + Time, + DateTime, + DateTimeWithTimeZone, +} + +enum ParsedTemporal { + Zoned(DateTime), + DateTime(NaiveDateTime), + Date(NaiveDate), + Time(NaiveTime), +} + +fn temporal_kind(data_type: Option<&str>) -> Option { + let normalized = data_type?.trim().to_ascii_lowercase().replace(char::is_whitespace, " "); + let base = normalized.split(['(', ':', ' ']).next().unwrap_or(""); + if matches!(base, "datetimeoffset" | "datetimeoffsetn" | "timestamptz") + || (base == "timestamp" + && (normalized.contains("with time zone") || normalized.contains("with local time zone"))) + { + return Some(TemporalKind::DateTimeWithTimeZone); + } + match base { + "date" | "date32" | "daten" => Some(TemporalKind::Date), + "time" | "time64" | "timen" | "timetz" => Some(TemporalKind::Time), + "datetime" | "datetime2" | "datetime4" | "datetime64" | "datetimen" | "smalldatetime" | "timestamp" + | "timestampdty" => Some(TemporalKind::DateTime), + _ if base.starts_with("timestamp_") => Some(TemporalKind::DateTime), + _ => None, + } +} + +fn dayjs_to_chrono_pattern(pattern: &str) -> Option { + let pattern = pattern.trim(); + if pattern.is_empty() || pattern.len() > 100 || pattern.contains('%') { + return None; + } + let tokens = [ + ("YYYY", "%Y"), + ("SSS", "%3f"), + ("ZZ", "%z"), + ("MM", "%m"), + ("DD", "%d"), + ("HH", "%H"), + ("mm", "%M"), + ("ss", "%S"), + ("M", "%-m"), + ("D", "%-d"), + ("H", "%-H"), + ("m", "%-M"), + ("s", "%-S"), + ("Z", "%:z"), + ]; + let mut output = String::with_capacity(pattern.len() * 2); + let mut index = 0; + while index < pattern.len() { + let remaining = &pattern[index..]; + if remaining.starts_with('[') { + let close = remaining.find(']')?; + output.push_str(&remaining[1..close]); + index += close + 1; + continue; + } + if let Some((token, replacement)) = tokens.iter().find(|(token, _)| remaining.starts_with(token)) { + output.push_str(replacement); + index += token.len(); + continue; + } + let ch = remaining.chars().next()?; + // Reject unknown Day.js tokens instead of silently exporting different text than the frontend displays. + if ch.is_ascii_alphabetic() { + return None; + } + output.push(ch); + index += ch.len_utf8(); + } + Some(output) +} + +fn parse_with_pattern(value: &str, pattern: &str) -> Option { + let pattern = dayjs_to_chrono_pattern(pattern)?; + DateTime::parse_from_str(value, &pattern) + .map(ParsedTemporal::Zoned) + .ok() + .or_else(|| NaiveDateTime::parse_from_str(value, &pattern).map(ParsedTemporal::DateTime).ok()) + .or_else(|| NaiveDate::parse_from_str(value, &pattern).map(ParsedTemporal::Date).ok()) + .or_else(|| NaiveTime::parse_from_str(value, &pattern).map(ParsedTemporal::Time).ok()) +} + +fn parse_known_temporal(value: &str) -> Option { + if let Ok(parsed) = DateTime::parse_from_rfc3339(value) { + return Some(ParsedTemporal::Zoned(parsed)); + } + for pattern in ["%Y-%m-%d %H:%M:%S%.f", "%Y-%m-%dT%H:%M:%S%.f", "%Y/%m/%d %H:%M:%S%.f", "%Y/%m/%dT%H:%M:%S%.f"] { + if let Ok(parsed) = NaiveDateTime::parse_from_str(value, pattern) { + return Some(ParsedTemporal::DateTime(parsed)); + } + } + for pattern in ["%Y-%m-%d", "%Y/%m/%d"] { + if let Ok(parsed) = NaiveDate::parse_from_str(value, pattern) { + return Some(ParsedTemporal::Date(parsed)); + } + } + for pattern in ["%H:%M:%S%.f", "%H:%M:%S"] { + if let Ok(parsed) = NaiveTime::parse_from_str(value, pattern) { + return Some(ParsedTemporal::Time(parsed)); + } + } + None +} + +fn parse_temporal(value: &str, preferred_pattern: Option<&str>) -> Option { + preferred_pattern + .filter(|pattern| !pattern.trim().is_empty()) + .and_then(|pattern| parse_with_pattern(value, pattern)) + .or_else(|| parse_known_temporal(value)) +} + +fn format_parsed(parsed: ParsedTemporal, pattern: &str) -> Option { + let pattern = dayjs_to_chrono_pattern(pattern)?; + let mut output = String::new(); + // Chrono reports missing date/time fields through fmt::Error; propagate it so exports preserve the raw value. + match parsed { + ParsedTemporal::Zoned(value) => write!(&mut output, "{}", value.format(&pattern)), + ParsedTemporal::DateTime(value) => write!(&mut output, "{}", value.format(&pattern)), + ParsedTemporal::Date(value) => write!(&mut output, "{}", value.format(&pattern)), + ParsedTemporal::Time(value) => write!(&mut output, "{}", value.format(&pattern)), + } + .ok()?; + Some(output) +} + +pub fn format_temporal_export_value(value: &Value, data_type: Option<&str>, pattern: Option<&str>) -> Value { + let Some(pattern) = pattern.filter(|pattern| !pattern.trim().is_empty()) else { + return value.clone(); + }; + if temporal_kind(data_type).is_none() { + return value.clone(); + } + let Some(raw) = value.as_str() else { + return value.clone(); + }; + parse_known_temporal(raw) + .and_then(|parsed| format_parsed(parsed, pattern)) + .map(Value::String) + .unwrap_or_else(|| value.clone()) +} + +pub fn format_temporal_export_row(row: &[Value], column_types: &[Option], pattern: Option<&str>) -> Vec { + row.iter() + .enumerate() + .map(|(index, value)| { + format_temporal_export_value( + value, + column_types.get(index).and_then(|data_type| data_type.as_deref()), + pattern, + ) + }) + .collect() +} + +pub fn format_temporal_export_rows( + rows: &[Vec], + column_types: &[Option], + pattern: Option<&str>, +) -> Vec> { + rows.iter().map(|row| format_temporal_export_row(row, column_types, pattern)).collect() +} + +pub fn format_temporal_export_row_with_string_types( + row: &[Value], + column_types: &[String], + pattern: Option<&str>, +) -> Vec { + row.iter() + .enumerate() + .map(|(index, value)| format_temporal_export_value(value, column_types.get(index).map(String::as_str), pattern)) + .collect() +} + +pub fn format_temporal_export_rows_with_string_types( + rows: &[Vec], + column_types: &[String], + pattern: Option<&str>, +) -> Vec> { + rows.iter().map(|row| format_temporal_export_row_with_string_types(row, column_types, pattern)).collect() +} + +pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, pattern: Option<&str>) -> Value { + let Some(kind) = temporal_kind(data_type) else { + return value.clone(); + }; + let Some(raw) = value.as_str() else { + return value.clone(); + }; + let Some(parsed) = parse_temporal(raw.trim(), pattern) else { + return value.clone(); + }; + + let normalized = match (kind, parsed) { + (TemporalKind::Date, ParsedTemporal::Zoned(value)) => value.date_naive().format("%Y-%m-%d").to_string(), + (TemporalKind::Date, ParsedTemporal::DateTime(value)) => value.date().format("%Y-%m-%d").to_string(), + (TemporalKind::Date, ParsedTemporal::Date(value)) => value.format("%Y-%m-%d").to_string(), + (TemporalKind::Date, ParsedTemporal::Time(_)) => return value.clone(), + (TemporalKind::Time, ParsedTemporal::Zoned(value)) => value.time().format("%H:%M:%S%.f").to_string(), + (TemporalKind::Time, ParsedTemporal::DateTime(value)) => value.time().format("%H:%M:%S%.f").to_string(), + (TemporalKind::Time, ParsedTemporal::Time(value)) => value.format("%H:%M:%S%.f").to_string(), + (TemporalKind::Time, ParsedTemporal::Date(_)) => return value.clone(), + (TemporalKind::DateTime, ParsedTemporal::Zoned(value)) => { + value.naive_local().format("%Y-%m-%d %H:%M:%S%.f").to_string() + } + (TemporalKind::DateTime, ParsedTemporal::DateTime(value)) => value.format("%Y-%m-%d %H:%M:%S%.f").to_string(), + (TemporalKind::DateTime, ParsedTemporal::Date(date)) => { + let Some(value) = date.and_hms_opt(0, 0, 0) else { + return value.clone(); + }; + value.format("%Y-%m-%d %H:%M:%S").to_string() + } + (TemporalKind::DateTime, ParsedTemporal::Time(_)) => return value.clone(), + (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Zoned(value)) => { + value.format("%Y-%m-%dT%H:%M:%S%.f%:z").to_string() + } + (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::DateTime(value)) => { + value.format("%Y-%m-%d %H:%M:%S%.f").to_string() + } + (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Date(date)) => { + let Some(value) = date.and_hms_opt(0, 0, 0) else { + return value.clone(); + }; + value.format("%Y-%m-%d %H:%M:%S").to_string() + } + (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Time(_)) => return value.clone(), + }; + Value::String(normalized) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn normalizes_unpadded_slash_dates_for_import() { + assert_eq!( + normalize_temporal_import_value(&json!("2024/2/25 13:02:15"), Some("DATE"), None), + json!("2024-02-25") + ); + assert_eq!( + normalize_temporal_import_value( + &json!("25.02.2024 13:02:15"), + Some("TIMESTAMP(6)"), + Some("DD.MM.YYYY HH:mm:ss") + ), + json!("2024-02-25 13:02:15") + ); + } + + #[test] + fn formats_only_typed_temporal_export_values() { + let row = vec![json!(1), json!("2024-02-25 13:02:15"), json!("2024-02-25 13:02:15")]; + assert_eq!( + format_temporal_export_row( + &row, + &[Some("NUMBER".into()), Some("TIMESTAMP".into()), Some("VARCHAR2".into())], + Some("YYYY/M/D HH:mm:ss") + ), + vec![json!(1), json!("2024/2/25 13:02:15"), json!("2024-02-25 13:02:15")] + ); + } + + #[test] + fn preserves_raw_export_values_when_pattern_requires_missing_fields() { + assert_eq!( + format_temporal_export_value(&json!("2024-02-25"), Some("DATE"), Some("YYYY-MM-DD HH:mm:ss")), + json!("2024-02-25") + ); + assert_eq!( + format_temporal_export_value(&json!("13:02:15"), Some("TIME"), Some("YYYY-MM-DD HH:mm:ss")), + json!("13:02:15") + ); + } + + #[test] + fn rejects_unsupported_dayjs_tokens_but_allows_literal_text() { + assert_eq!(dayjs_to_chrono_pattern("MM/DD/YYYY hh:mm A"), None); + assert_eq!(dayjs_to_chrono_pattern("YYYY-MM-DD [at] HH:mm:ss"), Some("%Y-%m-%d at %H:%M:%S".into())); + } + + #[test] + fn recognizes_common_driver_temporal_type_aliases() { + for data_type in ["DateTime64(3)", "date32", "timestamp_ns", "TimeStampDTY", "datetimeoffsetn", "timen"] { + assert!(temporal_kind(Some(data_type)).is_some(), "{data_type}"); + } + } + + #[test] + fn export_formatting_preserves_offset_datetime_fields() { + assert_eq!( + format_temporal_export_value(&json!("2024-02-25T13:02:15Z"), Some("DATE"), Some("YYYY/M/D HH:mm:ss")), + json!("2024/2/25 13:02:15") + ); + } + + #[test] + fn import_normalization_preserves_timezone_offsets() { + assert_eq!( + normalize_temporal_import_value( + &json!("2024-02-25T13:02:15+08:00"), + Some("timestamp with time zone"), + None + ), + json!("2024-02-25T13:02:15+08:00") + ); + } +} diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index 77370737b..42036ce67 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -1073,8 +1073,8 @@ pub fn escape_value_typed(val: &serde_json::Value, db_type: &DatabaseType, colum if let Some(numeric_literal) = format_mysql_numeric_string_literal(s, db_type, column_type) { return numeric_literal; } - if let Some(date_literal) = format_oracle_date_sql_literal(s, db_type, column_type) { - return date_literal; + if let Some(temporal_literal) = format_oracle_temporal_sql_literal(s, db_type, column_type) { + return temporal_literal; } let literal = format_literal_string(s, db_type, column_type); @@ -1178,21 +1178,46 @@ fn format_mysql_binary_sql_literal(value: &str, db_type: &DatabaseType, column_t } } -fn format_oracle_date_sql_literal(value: &str, db_type: &DatabaseType, column_type: Option<&str>) -> Option { +fn format_oracle_temporal_sql_literal( + value: &str, + db_type: &DatabaseType, + column_type: Option<&str>, +) -> Option { if !matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle) { return None; } - if temporal_column_kind(column_type) != Some("date") { - return None; - } + let kind = temporal_column_kind(column_type)?; + let normalized_column_type = column_type?.trim().to_ascii_lowercase(); let parts = oracle_export_date_parts(value)?; - Some(format_oracle_date_sql_literal_parts(&parts)) + match kind { + "date" => Some(format_oracle_date_sql_literal_parts(&parts)), + "datetime" + if (normalized_column_type.contains("with time zone") + || normalized_column_type.contains("with local time zone")) + && parts.zone.is_some() => + { + let fraction = parts.fraction.unwrap_or_default(); + let mask = if fraction.is_empty() { "YYYY-MM-DD HH24:MI:SS" } else { "YYYY-MM-DD HH24:MI:SS.FF" }; + let zone = match parts.zone.unwrap_or_default() { + "Z" | "z" => "+00:00", + zone => zone, + }; + Some(format!("TO_TIMESTAMP_TZ('{} {}{fraction} {zone}', '{mask} TZH:TZM')", parts.date, parts.time)) + } + "datetime" => { + let fraction = parts.fraction.unwrap_or_default(); + let mask = if fraction.is_empty() { "YYYY-MM-DD HH24:MI:SS" } else { "YYYY-MM-DD HH24:MI:SS.FF" }; + Some(format!("TO_TIMESTAMP('{} {}{fraction}', '{mask}')", parts.date, parts.time)) + } + _ => None, + } } struct OracleExportDateParts<'a> { date: &'a str, time: &'a str, fraction: Option<&'a str>, + zone: Option<&'a str>, } fn format_oracle_date_sql_literal_parts(parts: &OracleExportDateParts<'_>) -> String { @@ -1218,7 +1243,7 @@ fn oracle_export_date_parts(value: &str) -> Option> { return None; } if bytes.len() == 10 { - return Some(OracleExportDateParts { date, time: "00:00:00", fraction: None }); + return Some(OracleExportDateParts { date, time: "00:00:00", fraction: None, zone: None }); } let separator = *bytes.get(10)?; if separator != b'T' && separator != b' ' { @@ -1233,7 +1258,7 @@ fn oracle_export_date_parts(value: &str) -> Option> { } let rest = &value[19..]; if rest.is_empty() || is_timezone_suffix(rest) { - return Some(OracleExportDateParts { date, time, fraction: None }); + return Some(OracleExportDateParts { date, time, fraction: None, zone: (!rest.is_empty()).then_some(rest) }); } if let Some(after_dot) = rest.strip_prefix('.') { let digit_count = after_dot.bytes().take_while(|byte| byte.is_ascii_digit()).count(); @@ -1242,7 +1267,12 @@ fn oracle_export_date_parts(value: &str) -> Option> { } let zone = &after_dot[digit_count..]; if zone.is_empty() || is_timezone_suffix(zone) { - return Some(OracleExportDateParts { date, time, fraction: Some(&value[19..19 + 1 + digit_count]) }); + return Some(OracleExportDateParts { + date, + time, + fraction: Some(&value[19..19 + 1 + digit_count]), + zone: (!zone.is_empty()).then_some(zone), + }); } } None @@ -5987,12 +6017,20 @@ mod tests { assert_eq!( sql, - "INSERT INTO \"APP\".\"events\" (\"id\", \"created_on\", \"created_at\", \"raw_text\") VALUES\n(1, TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), '2022-08-25T09:58:43Z', '2022-08-25T09:58:43Z')" + "INSERT INTO \"APP\".\"events\" (\"id\", \"created_on\", \"created_at\", \"raw_text\") VALUES\n(1, TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), TO_TIMESTAMP('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), '2022-08-25T09:58:43Z')" ); assert_eq!( escape_value_typed(&json!("2022-08-25T00:00:00Z"), &DatabaseType::Oracle, Some("DATE")), "DATE '2022-08-25'" ); + assert_eq!( + escape_value_typed( + &json!("2022-08-25T09:58:43.123456+08:00"), + &DatabaseType::Oracle, + Some("TIMESTAMP(6) WITH TIME ZONE") + ), + "TO_TIMESTAMP_TZ('2022-08-25 09:58:43.123456 +08:00', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')" + ); } #[test] diff --git a/crates/dbx-core/tests/live_clickhouse_query_result_export.rs b/crates/dbx-core/tests/live_clickhouse_query_result_export.rs index 61b6b1276..e1d87621f 100644 --- a/crates/dbx-core/tests/live_clickhouse_query_result_export.rs +++ b/crates/dbx-core/tests/live_clickhouse_query_result_export.rs @@ -98,6 +98,7 @@ async fn live_clickhouse_query_result_export_xlsx_streams_random_order_query_onc keyset_optimization_enabled: false, client_session_id: None, execution_id: Some(format!("live-clickhouse-query-export-{suffix}")), + date_time_format: None, }; let done_seen = AtomicBool::new(false); let result = export_query_result_core(&state, &request, None, |progress| { diff --git a/crates/dbx-core/tests/live_mysql57.rs b/crates/dbx-core/tests/live_mysql57.rs index 52a634af1..68f060851 100644 --- a/crates/dbx-core/tests/live_mysql57.rs +++ b/crates/dbx-core/tests/live_mysql57.rs @@ -165,6 +165,7 @@ async fn live_mysql_query_result_export_xlsx_streams_single_query_without_duplic keyset_optimization_enabled: false, client_session_id: None, execution_id: Some(format!("live-mysql-query-export-{suffix}")), + date_time_format: None, }; let done_seen = AtomicBool::new(false); let result = export_query_result_core(&state, &request, None, |progress| { diff --git a/crates/dbx-core/tests/live_postgres_query_result_export.rs b/crates/dbx-core/tests/live_postgres_query_result_export.rs index f5c2a606f..dda513b8d 100644 --- a/crates/dbx-core/tests/live_postgres_query_result_export.rs +++ b/crates/dbx-core/tests/live_postgres_query_result_export.rs @@ -130,6 +130,7 @@ async fn live_postgres_query_result_export_uses_single_streamed_query() { keyset_optimization_enabled: true, client_session_id: None, execution_id: Some(format!("live-postgres-query-export-{suffix}")), + date_time_format: None, }; let done_seen = AtomicBool::new(false); let result = export_query_result_core(&state, &request, None, |progress| { diff --git a/crates/dbx-core/tests/live_sqlserver_completion.rs b/crates/dbx-core/tests/live_sqlserver_completion.rs index ec02cee6e..458509d93 100644 --- a/crates/dbx-core/tests/live_sqlserver_completion.rs +++ b/crates/dbx-core/tests/live_sqlserver_completion.rs @@ -279,7 +279,7 @@ async fn live_sqlserver_stream_first_result_set_exports_cte_query_rows() { let mut rows = Vec::new(); let result = dbx_core::db::sqlserver::stream_first_result_set(&mut client, &sql, None, None, |item| { match item { - dbx_core::db::sqlserver::SqlServerStreamItem::Columns(stream_columns) => { + dbx_core::db::sqlserver::SqlServerStreamItem::Columns { columns: stream_columns, .. } => { columns = stream_columns.to_vec(); } dbx_core::db::sqlserver::SqlServerStreamItem::Row(row) => { @@ -369,6 +369,7 @@ async fn live_sqlserver_query_result_export_streams_cte_query_to_csv() { keyset_optimization_enabled: true, client_session_id: None, execution_id: Some(format!("live-sqlserver-export-{suffix}")), + date_time_format: None, }; let done_seen = AtomicBool::new(false); let result = export_query_result_core(&state, &request, None, |progress| { diff --git a/packages/app-tests/columnFormatter.test.ts b/packages/app-tests/columnFormatter.test.ts index 774e70131..df5d97932 100644 --- a/packages/app-tests/columnFormatter.test.ts +++ b/packages/app-tests/columnFormatter.test.ts @@ -17,6 +17,27 @@ test("formats unix timestamps in seconds, milliseconds, and auto mode", () => { assert.equal(applyColumnFormatter("1715758200", { kind: "datetime", unit: "auto", pattern: "YYYY-MM-DD HH:mm:ssZ" }), "2024-05-15 15:30:00+08:00"); }); +test("uses the global datetime formatter only for temporal columns without a column override", () => { + const global = { pattern: "YYYY/MM/DD HH:mm:ss", columnType: "TIMESTAMP(6)" }; + + assert.deepEqual(resolveColumnFormatter(undefined, {}, global), { kind: "datetime", unit: "auto", pattern: "YYYY/MM/DD HH:mm:ss" }); + assert.equal(resolveColumnFormatter(undefined, {}, { ...global, columnType: "VARCHAR2(100)" }), undefined); + assert.deepEqual(resolveColumnFormatter({ kind: "mask", prefix: 2, suffix: 2 }, {}, global), { kind: "mask", prefix: 2, suffix: 2 }); + assert.deepEqual(resolveColumnFormatter(undefined, {}, { ...global, columnType: "DateTime64(3)" }), { kind: "datetime", unit: "auto", pattern: global.pattern }); + assert.deepEqual(resolveColumnFormatter(undefined, {}, { ...global, columnType: "datetimeoffsetn" }), { kind: "datetime", unit: "auto", pattern: global.pattern }); +}); + +test("formats only typed temporal cells for export", async () => { + const { formatTemporalRowsForExport } = await import("../../apps/desktop/src/lib/dataGrid/columnFormatter.ts"); + const rows = formatTemporalRowsForExport([[1, "2024-02-25T05:02:15Z", "2024-02-25T05:02:15Z", "2024-02-25T05:02:15.123456+08:00", "2024-02-25 13:02:15.987654"]], ["NUMBER", "TIMESTAMP", "VARCHAR2", "TIMESTAMP WITH TIME ZONE", "TIMESTAMP(6)"], "YYYY/MM/DD HH:mm:ss.SSSZ"); + + assert.deepEqual(rows, [[1, "2024/02/25 05:02:15.000+00:00", "2024-02-25T05:02:15Z", "2024/02/25 05:02:15.123+08:00", "2024/02/25 13:02:15.987+08:00"]]); +}); + +test("formats Oracle timestamp fractional precision in the data grid", () => { + assert.equal(applyColumnFormatter("2024-02-25 13:02:15.123456", { kind: "datetime", unit: "auto", pattern: "YYYY/MM/DD HH:mm:ss.SSS" }), "2024/02/25 13:02:15.123"); +}); + test("does not treat compact date strings as unix timestamps", () => { dayjs.extend(utc); dayjs.extend(timezone); diff --git a/packages/app-tests/dataGridExport.test.ts b/packages/app-tests/dataGridExport.test.ts index 10b70670f..c57a8c842 100644 --- a/packages/app-tests/dataGridExport.test.ts +++ b/packages/app-tests/dataGridExport.test.ts @@ -1,6 +1,7 @@ import { strict as assert } from "node:assert"; import { beforeEach, test, vi } from "vitest"; import { computed, ref } from "vue"; +import { createPinia, setActivePinia } from "pinia"; import { useDataGridExport } from "../../apps/desktop/src/composables/useDataGridExport.ts"; import { copyToClipboard } from "@/lib/common/clipboard"; import * as api from "@/lib/backend/api"; @@ -21,11 +22,13 @@ vi.mock("@/lib/backend/api", () => ({ const draftRowId = Number.MIN_SAFE_INTEGER; -function createExportContext(options: { - contextRowId?: number; - selectedRowIds?: Set; - fullExportResult?: () => Promise<{ columns: string[]; rows: Array>; affected_rows: number; execution_time_ms: number }>; -} = {}) { +function createExportContext( + options: { + contextRowId?: number; + selectedRowIds?: Set; + fullExportResult?: () => Promise<{ columns: string[]; rows: Array>; affected_rows: number; execution_time_ms: number }>; + } = {}, +) { const contextRowId = options.contextRowId ?? draftRowId; const selectedRowIds = ref(options.selectedRowIds ?? new Set()); const rows = [ @@ -75,6 +78,7 @@ function createExportContext(options: { } beforeEach(() => { + setActivePinia(createPinia()); globalThis.window = { setTimeout, clearTimeout, diff --git a/packages/app-tests/settingsStore.test.ts b/packages/app-tests/settingsStore.test.ts index 82078f7c4..ccc4c08c2 100644 --- a/packages/app-tests/settingsStore.test.ts +++ b/packages/app-tests/settingsStore.test.ts @@ -342,6 +342,19 @@ test("defaults column formatters to an empty record", () => { assert.deepEqual(normalizeEditorSettings({}).columnFormatters, {}); }); +test("normalizes global datetime display and transfer formats", () => { + assert.equal(DEFAULT_EDITOR_SETTINGS.globalDateTimeDisplayFormat, ""); + const settings = normalizeEditorSettings({ + globalDateTimeDisplayFormat: " YYYY/MM/DD HH:mm:ss ", + globalDateTimeExportFormat: "YYYY-M-D H:m:s", + globalDateTimeImportFormat: 123, + } as any); + + assert.equal(settings.globalDateTimeDisplayFormat, "YYYY/MM/DD HH:mm:ss"); + assert.equal(settings.globalDateTimeExportFormat, "YYYY-M-D H:m:s"); + assert.equal(settings.globalDateTimeImportFormat, ""); +}); + test("keeps only valid saved column formatter configs", () => { const settings = normalizeEditorSettings({ columnFormatters: { diff --git a/packages/app-tests/useDataGridExport.test.ts b/packages/app-tests/useDataGridExport.test.ts index f123aff8e..f6b3a3158 100644 --- a/packages/app-tests/useDataGridExport.test.ts +++ b/packages/app-tests/useDataGridExport.test.ts @@ -52,10 +52,18 @@ function buildExportHarness( options: { currentResultLabel?: string; exportFileBaseName?: string; + columns?: string[]; columnTypes?: Array; + rows?: QueryResult["rows"]; allExportResults?: Array<{ sheetName: string; result: QueryResult; sql?: string }>; } = {}, ) { + const exportColumns = options.columns ?? ["id", "name"]; + const exportRows = options.rows ?? [ + [1, "Ada"], + [2, "Lin"], + ]; + const rowItems = exportRows.map((data, index) => ({ id: index + 1, data, isNew: false, isDeleted: false, isDirtyCol: data.map(() => false), status: "" })); const exportProgressDialog = ref(false); const exportProgressState = ref({ title: "", @@ -93,11 +101,8 @@ function buildExportHarness( })); const composable = useDataGridExport({ - columns: computed(() => ["id", "name"]), - displayItems: computed(() => [ - { id: 1, data: [1, "Ada"], isNew: false, isDeleted: false, isDirtyCol: [false, false], status: "" }, - { id: 2, data: [2, "Lin"], isNew: false, isDeleted: false, isDirtyCol: [false, false], status: "" }, - ]), + columns: computed(() => exportColumns), + displayItems: computed(() => rowItems), sql: computed(() => "SELECT * FROM users"), exportSql: computed(() => "SELECT * FROM users ORDER BY id DESC"), tableMeta: computed(() => undefined), @@ -114,11 +119,7 @@ function buildExportHarness( selectedCells: computed(() => ({ columns: [], rows: [] })), selectedRange: computed(() => null), contextCell: ref(null), - getRowItem: (rowId: number) => - [ - { id: 1, data: [1, "Ada"], isNew: false, isDeleted: false, isDirtyCol: [false, false], status: "" }, - { id: 2, data: [2, "Lin"], isNew: false, isDeleted: false, isDirtyCol: [false, false], status: "" }, - ].find((item) => item.id === rowId), + getRowItem: (rowId: number) => rowItems.find((item) => item.id === rowId), selectedRowIds: ref(new Set()), hasRowSelection: computed(() => false), fullExportResult, @@ -652,6 +653,7 @@ test("default data grid export file names use sanitized base names and compact l }); test("full query result CSV export streams through the backend without loading all rows", async () => { + useSettingsStore().updateEditorSettings({ globalDateTimeExportFormat: "YYYY/M/D HH:mm:ss" }); const { composable, fullExportResult, queryResultExportRequest, exportProgressDialog, exportProgressState } = buildExportHarness(); await composable.exportCsv(); @@ -659,6 +661,7 @@ test("full query result CSV export streams through the backend without loading a assert.equal(fullExportResult.mock.calls.length, 0); assert.equal(queryResultExportRequest.mock.calls.length, 1); assert.equal(apiMock.startQueryResultExport.mock.calls.length, 1); + assert.equal(apiMock.startQueryResultExport.mock.calls[0][0].dateTimeFormat, "YYYY/M/D HH:mm:ss"); assert.equal(apiMock.exportQueryResultCsv.mock.calls.length, 0); assert.equal(exportProgressDialog.value, true); assert.equal(exportProgressState.value.status, "Done"); @@ -773,6 +776,20 @@ test("selected query result CSV export keeps the existing in-memory path", async assert.deepEqual(apiMock.exportQueryResultCsv.mock.calls[0][2], [[1, "Ada"]]); }); +test("selected query result CSV export formats only typed temporal columns", async () => { + useSettingsStore().updateEditorSettings({ globalDateTimeExportFormat: "YYYY/M/D HH:mm:ss" }); + const rawDateTime = "2024-02-25 13:02:15"; + const { composable } = buildExportHarness({ + columns: ["created_at", "note"], + columnTypes: ["timestamp", "varchar"], + rows: [[rawDateTime, rawDateTime]], + }); + + await composable.exportCsv([1]); + + assert.deepEqual(apiMock.exportQueryResultCsv.mock.calls[0][2], [["2024/2/25 13:02:15", rawDateTime]]); +}); + test("selected query result XLSX export uses the current source label as the sheet name", async () => { const { composable, queryResultExportRequest } = buildExportHarness({ currentResultLabel: "aaa.apis", columnTypes: ["bigint(20)", "varchar(64)"] }); @@ -888,6 +905,20 @@ test("table data export leaves row limit unset by default", async () => { } }); +test("table data export passes the global date time export format", async () => { + const restoreStorage = installMemoryStorage(); + try { + useSettingsStore().updateEditorSettings({ globalDateTimeExportFormat: "YYYY/MM/DD HH:mm:ss" }); + const { composable } = buildTableDataExportHarness(); + + await composable.exportCsv(); + + assert.equal(apiMock.startTableExport.mock.calls[0][0].dateTimeFormat, "YYYY/MM/DD HH:mm:ss"); + } finally { + restoreStorage(); + } +}); + test("table data export requests row count for determinate progress", async () => { const restoreStorage = installMemoryStorage(); try {