feat(import): support text encoding selection
This commit is contained in:
zipg
GitHub
parent
32b4acee3a
commit
95e50a0da7
|
|
@ -72,6 +72,7 @@ const errorMessage = ref("");
|
|||
const wizardStep = ref<TableImportWizardStep>("source");
|
||||
const fileInput = ref<HTMLInputElement | null>(null);
|
||||
const delimiter = ref(",");
|
||||
const textEncoding = ref<api.TableImportTextEncoding>("auto");
|
||||
const titleRow = ref(1);
|
||||
const dataStartRow = ref(2);
|
||||
const lastDataRow = ref(0);
|
||||
|
|
@ -82,6 +83,8 @@ const jsonShape = ref<api.TableImportJsonShape>("auto");
|
|||
const previewLimit = ref(50);
|
||||
let previewReloadTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let dataTypeOptionsRequestId = 0;
|
||||
let previewRequestId = 0;
|
||||
let batchEncodingRequestId = 0;
|
||||
|
||||
const formatOptions: Array<{ value: api.TableImportSourceFormat; icon: any; labelKey: string; descriptionKey: string }> = [
|
||||
{ value: "csv", icon: FileText, labelKey: "tableImport.formatCsv", descriptionKey: "tableImport.formatCsvDescription" },
|
||||
|
|
@ -91,6 +94,14 @@ const formatOptions: Array<{ value: api.TableImportSourceFormat; icon: any; labe
|
|||
{ value: "excel", icon: FileSpreadsheet, labelKey: "tableImport.formatExcel", descriptionKey: "tableImport.formatExcelDescription" },
|
||||
];
|
||||
|
||||
const encodingOptions: Array<{ value: api.TableImportTextEncoding; labelKey: string }> = [
|
||||
{ value: "auto", labelKey: "tableImport.encodingAuto" },
|
||||
{ value: "utf8", labelKey: "tableImport.encodingUtf8" },
|
||||
{ value: "gbk", labelKey: "tableImport.encodingGbk" },
|
||||
{ value: "utf16Le", labelKey: "tableImport.encodingUtf16Le" },
|
||||
{ value: "utf16Be", labelKey: "tableImport.encodingUtf16Be" },
|
||||
];
|
||||
|
||||
const wizardSteps: Array<{ value: TableImportWizardStep; labelKey: string }> = [
|
||||
{ value: "source", labelKey: "tableImport.stepSource" },
|
||||
{ value: "options", labelKey: "tableImport.stepOptions" },
|
||||
|
|
@ -169,6 +180,7 @@ const createColumnSummaries = computed(() =>
|
|||
);
|
||||
const parseOptions = computed<api.TableImportParseOptions>(() => ({
|
||||
delimiter: sourceFormat.value === "tsv" ? "\\t" : sourceFormat.value === "csv" ? "," : delimiter.value,
|
||||
encoding: isDelimitedFormat(sourceFormat.value) ? textEncoding.value : null,
|
||||
titleRow: titleRow.value,
|
||||
dataStartRow: dataStartRow.value,
|
||||
lastDataRow: lastDataRow.value,
|
||||
|
|
@ -180,6 +192,12 @@ const parseOptions = computed<api.TableImportParseOptions>(() => ({
|
|||
const terminalStatus = computed(() => progress.value?.status && ["done", "error", "cancelled"].includes(progress.value.status));
|
||||
|
||||
function resetState() {
|
||||
previewRequestId++;
|
||||
batchEncodingRequestId++;
|
||||
if (previewReloadTimer) {
|
||||
clearTimeout(previewReloadTimer);
|
||||
previewReloadTimer = null;
|
||||
}
|
||||
targetColumns.value = [];
|
||||
targetMode.value = props.prefillTable ? "existing" : "create";
|
||||
newTableName.value = "";
|
||||
|
|
@ -188,6 +206,7 @@ function resetState() {
|
|||
activeTaskIndex.value = 0;
|
||||
sourceFormat.value = "csv";
|
||||
delimiter.value = ",";
|
||||
textEncoding.value = "auto";
|
||||
titleRow.value = 1;
|
||||
dataStartRow.value = 2;
|
||||
lastDataRow.value = 0;
|
||||
|
|
@ -201,6 +220,7 @@ function resetState() {
|
|||
columnDataTypes.value = {};
|
||||
importMode.value = "append";
|
||||
batchSize.value = 500;
|
||||
loadingPreview.value = false;
|
||||
running.value = false;
|
||||
cancelling.value = false;
|
||||
importId.value = "";
|
||||
|
|
@ -218,6 +238,14 @@ function detectFormat(name: string): api.TableImportSourceFormat {
|
|||
return "csv";
|
||||
}
|
||||
|
||||
function isDelimitedFormat(format: api.TableImportSourceFormat) {
|
||||
return format === "csv" || format === "tsv" || format === "delimited";
|
||||
}
|
||||
|
||||
function encodingLabel(encoding: api.TableImportTextEncoding) {
|
||||
return t(encodingOptions.find((option) => option.value === encoding)?.labelKey || "tableImport.encodingAuto");
|
||||
}
|
||||
|
||||
function suggestedTableName(name: string) {
|
||||
const baseName = name.split(/[\\/]/).pop() || name;
|
||||
const withoutExtension = baseName.replace(/\.[^.]+$/, "").trim();
|
||||
|
|
@ -240,6 +268,7 @@ function uniqueTableName(baseName: string, usedNames: Set<string>): string {
|
|||
function taskParseOptions(format: api.TableImportSourceFormat, sheetName = ""): api.TableImportParseOptions {
|
||||
return {
|
||||
delimiter: format === "tsv" ? "\\t" : format === "csv" ? "," : delimiter.value,
|
||||
encoding: isDelimitedFormat(format) ? textEncoding.value : null,
|
||||
titleRow: titleRow.value,
|
||||
dataStartRow: dataStartRow.value,
|
||||
lastDataRow: lastDataRow.value,
|
||||
|
|
@ -250,6 +279,14 @@ function taskParseOptions(format: api.TableImportSourceFormat, sheetName = ""):
|
|||
};
|
||||
}
|
||||
|
||||
function importParseOptions(format: api.TableImportSourceFormat, currentPreview: api.TableImportPreview, sheetName = ""): api.TableImportParseOptions {
|
||||
const options = taskParseOptions(format, sheetName);
|
||||
if (isDelimitedFormat(format) && options.encoding === "auto" && currentPreview.effectiveEncoding) {
|
||||
options.encoding = currentPreview.effectiveEncoding;
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function mergeDataTypeOptions(...groups: readonly string[][]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const result: string[] = [];
|
||||
|
|
@ -337,10 +374,12 @@ async function previewSelectedImportFile(fileOrPath: string | File) {
|
|||
|
||||
async function loadPreview(fileOrPath = selectedSource.value) {
|
||||
if (!fileOrPath) return;
|
||||
const requestId = ++previewRequestId;
|
||||
loadingPreview.value = true;
|
||||
errorMessage.value = "";
|
||||
try {
|
||||
const nextPreview = await previewSelectedImportFile(fileOrPath);
|
||||
if (requestId !== previewRequestId) return;
|
||||
preview.value = nextPreview;
|
||||
if (sourceFormat.value === "excel" && !selectedSheet.value && nextPreview.sheets?.length) {
|
||||
selectedSheet.value = nextPreview.sheets[0];
|
||||
|
|
@ -348,12 +387,13 @@ async function loadPreview(fileOrPath = selectedSource.value) {
|
|||
applyAutoMapping();
|
||||
applySuggestedColumnDataTypes(nextPreview);
|
||||
} catch (e: any) {
|
||||
if (requestId !== previewRequestId) return;
|
||||
preview.value = null;
|
||||
columnMapping.value = {};
|
||||
columnDataTypes.value = {};
|
||||
errorMessage.value = String(e?.message || e);
|
||||
} finally {
|
||||
loadingPreview.value = false;
|
||||
if (requestId === previewRequestId) loadingPreview.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -580,7 +620,7 @@ async function startImport() {
|
|||
filePath: currentPreview.filePath,
|
||||
sourceRef: currentPreview.sourceRef || null,
|
||||
sourceFormat: sourceFormat.value,
|
||||
parseOptions: parseOptions.value,
|
||||
parseOptions: importParseOptions(sourceFormat.value, currentPreview),
|
||||
mappings: mappedColumns.value,
|
||||
mode: targetMode.value === "create" ? "append" : importMode.value,
|
||||
createTable: targetMode.value === "create",
|
||||
|
|
@ -655,7 +695,7 @@ async function startBatchImport() {
|
|||
filePath: task.preview.filePath,
|
||||
sourceRef: task.preview.sourceRef || null,
|
||||
sourceFormat: task.format,
|
||||
parseOptions: taskParseOptions(task.format, task.sheetName),
|
||||
parseOptions: importParseOptions(task.format, task.preview, task.sheetName),
|
||||
mappings,
|
||||
mode: "append",
|
||||
createTable: true,
|
||||
|
|
@ -712,6 +752,48 @@ function schedulePreviewReload() {
|
|||
}, 250);
|
||||
}
|
||||
|
||||
function schedulePreviewReloadAfterEncodingChange() {
|
||||
if (isBatchImport.value) {
|
||||
void reloadBatchPreviewsForEncoding();
|
||||
return;
|
||||
}
|
||||
if (!selectedSource.value || running.value) return;
|
||||
previewRequestId++;
|
||||
if (previewReloadTimer) clearTimeout(previewReloadTimer);
|
||||
previewReloadTimer = setTimeout(() => {
|
||||
void loadPreview();
|
||||
}, 250);
|
||||
}
|
||||
|
||||
async function reloadBatchPreviewsForEncoding() {
|
||||
if (!isBatchImport.value || running.value) return;
|
||||
const requestId = ++batchEncodingRequestId;
|
||||
loadingPreview.value = true;
|
||||
errorMessage.value = "";
|
||||
try {
|
||||
for (const task of batchTasks.value) {
|
||||
if (!isDelimitedFormat(task.format)) continue;
|
||||
const nextPreview = await api.previewTableImportFile(task.source, {
|
||||
sourceFormat: task.format,
|
||||
parseOptions: taskParseOptions(task.format, task.sheetName),
|
||||
previewLimit: Math.max(1, Number(previewLimit.value) || 50),
|
||||
});
|
||||
if (requestId !== batchEncodingRequestId) return;
|
||||
task.preview = nextPreview;
|
||||
task.columnMapping = Object.fromEntries(nextPreview.columns.map((column) => [column, column]));
|
||||
task.columnDataTypes = suggestImportTargetDataTypes(nextPreview.columns, nextPreview.rows, structureDatabaseType.value);
|
||||
}
|
||||
activateBatchTask(activeTaskIndex.value);
|
||||
} catch (e: any) {
|
||||
if (requestId === batchEncodingRequestId) {
|
||||
preview.value = null;
|
||||
errorMessage.value = String(e?.message || e);
|
||||
}
|
||||
} finally {
|
||||
if (requestId === batchEncodingRequestId) loadingPreview.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
watch(
|
||||
open,
|
||||
(value) => {
|
||||
|
|
@ -725,6 +807,7 @@ watch(
|
|||
);
|
||||
|
||||
watch([sourceFormat, delimiter, titleRow, dataStartRow, lastDataRow, trimValues, emptyStringAsNull, selectedSheet, jsonShape, previewLimit], schedulePreviewReload);
|
||||
watch(textEncoding, schedulePreviewReloadAfterEncodingChange);
|
||||
watch([newTableName, columnMapping, columnDataTypes], saveActiveBatchTask, { deep: true });
|
||||
watch(targetMode, (mode) => {
|
||||
if (mode === "existing") {
|
||||
|
|
@ -886,7 +969,23 @@ watch(targetMode, (mode) => {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="sourceFormat === 'csv' || sourceFormat === 'tsv' || sourceFormat === 'delimited'" class="grid grid-cols-4 gap-3 rounded-md border p-3">
|
||||
<div v-if="sourceFormat === 'csv' || sourceFormat === 'tsv' || sourceFormat === 'delimited'" class="grid grid-cols-5 gap-3 rounded-md border p-3">
|
||||
<div class="space-y-1.5">
|
||||
<Label class="text-xs">{{ t("tableImport.encoding") }}</Label>
|
||||
<Select :model-value="textEncoding" @update:model-value="(value: any) => (textEncoding = value)">
|
||||
<SelectTrigger class="h-8 text-xs">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem v-for="option in encodingOptions" :key="option.value" :value="option.value">
|
||||
{{ t(option.labelKey) }}
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<div v-if="textEncoding === 'auto' && preview?.effectiveEncoding" class="truncate text-[11px] text-muted-foreground" :title="t('tableImport.encodingDetected', { encoding: encodingLabel(preview.effectiveEncoding) })">
|
||||
{{ t("tableImport.encodingDetected", { encoding: encodingLabel(preview.effectiveEncoding) }) }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="space-y-1.5">
|
||||
<Label class="text-xs">{{ t("tableImport.delimiter") }}</Label>
|
||||
<Input v-model="delimiter" :disabled="sourceFormat !== 'delimited'" class="h-8 text-xs font-mono" />
|
||||
|
|
|
|||
|
|
@ -2619,6 +2619,13 @@ export default {
|
|||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Encoding",
|
||||
encodingAuto: "Auto-detect",
|
||||
encodingUtf8: "UTF-8",
|
||||
encodingGbk: "GBK / GB18030",
|
||||
encodingUtf16Le: "UTF-16 LE",
|
||||
encodingUtf16Be: "UTF-16 BE",
|
||||
encodingDetected: "Detected: {encoding}",
|
||||
hasHeader: "First row is header",
|
||||
titleRow: "Title row (0 for none)",
|
||||
dataStartRow: "Data start row",
|
||||
|
|
|
|||
|
|
@ -2450,6 +2450,13 @@ export default withEnglishFallback({
|
|||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Codificación",
|
||||
encodingAuto: "Detección automática",
|
||||
encodingUtf8: "UTF-8",
|
||||
encodingGbk: "GBK / GB18030",
|
||||
encodingUtf16Le: "UTF-16 LE",
|
||||
encodingUtf16Be: "UTF-16 BE",
|
||||
encodingDetected: "Detectada: {encoding}",
|
||||
hasHeader: "First row is header",
|
||||
trimValues: "Trim values",
|
||||
emptyStringAsNull: "Empty string as NULL",
|
||||
|
|
|
|||
|
|
@ -2448,6 +2448,13 @@ export default withEnglishFallback({
|
|||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Codifica",
|
||||
encodingAuto: "Rilevamento automatico",
|
||||
encodingUtf8: "UTF-8",
|
||||
encodingGbk: "GBK / GB18030",
|
||||
encodingUtf16Le: "UTF-16 LE",
|
||||
encodingUtf16Be: "UTF-16 BE",
|
||||
encodingDetected: "Rilevata: {encoding}",
|
||||
hasHeader: "First row is header",
|
||||
trimValues: "Trim values",
|
||||
emptyStringAsNull: "Empty string as NULL",
|
||||
|
|
|
|||
|
|
@ -2449,6 +2449,13 @@ export default withEnglishFallback({
|
|||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "文字コード",
|
||||
encodingAuto: "自動検出",
|
||||
encodingUtf8: "UTF-8",
|
||||
encodingGbk: "GBK / GB18030",
|
||||
encodingUtf16Le: "UTF-16 LE",
|
||||
encodingUtf16Be: "UTF-16 BE",
|
||||
encodingDetected: "検出: {encoding}",
|
||||
hasHeader: "First row is header",
|
||||
trimValues: "Trim values",
|
||||
emptyStringAsNull: "Empty string as NULL",
|
||||
|
|
|
|||
|
|
@ -2450,6 +2450,13 @@ export default withEnglishFallback({
|
|||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Codificação",
|
||||
encodingAuto: "Detecção automática",
|
||||
encodingUtf8: "UTF-8",
|
||||
encodingGbk: "GBK / GB18030",
|
||||
encodingUtf16Le: "UTF-16 LE",
|
||||
encodingUtf16Be: "UTF-16 BE",
|
||||
encodingDetected: "Detectada: {encoding}",
|
||||
hasHeader: "First row is header",
|
||||
trimValues: "Trim values",
|
||||
emptyStringAsNull: "Empty string as NULL",
|
||||
|
|
|
|||
|
|
@ -2618,6 +2618,13 @@ export default withEnglishFallback({
|
|||
formatExcelDescription: "XLS、XLSX、XLSM",
|
||||
previewRows: "预览行数",
|
||||
delimiter: "分隔符",
|
||||
encoding: "文件编码",
|
||||
encodingAuto: "自动检测",
|
||||
encodingUtf8: "UTF-8",
|
||||
encodingGbk: "GBK / GB18030",
|
||||
encodingUtf16Le: "UTF-16 LE",
|
||||
encodingUtf16Be: "UTF-16 BE",
|
||||
encodingDetected: "检测结果：{encoding}",
|
||||
hasHeader: "首行为表头",
|
||||
titleRow: "标题行（0 表示无）",
|
||||
dataStartRow: "数据开始行",
|
||||
|
|
|
|||
|
|
@ -2305,6 +2305,13 @@ export default withEnglishFallback({
|
|||
formatExcelDescription: "XLS、XLSX、XLSM",
|
||||
previewRows: "預覽列數",
|
||||
delimiter: "分隔符",
|
||||
encoding: "檔案編碼",
|
||||
encodingAuto: "自動偵測",
|
||||
encodingUtf8: "UTF-8",
|
||||
encodingGbk: "GBK / GB18030",
|
||||
encodingUtf16Le: "UTF-16 LE",
|
||||
encodingUtf16Be: "UTF-16 BE",
|
||||
encodingDetected: "偵測結果：{encoding}",
|
||||
hasHeader: "首列為標頭",
|
||||
trimValues: "裁剪空白",
|
||||
emptyStringAsNull: "空字串作為 NULL",
|
||||
|
|
|
|||
|
|
@ -556,6 +556,7 @@ export type {
|
|||
TableImportStatus,
|
||||
TableImportSourceFormat,
|
||||
TableImportJsonShape,
|
||||
TableImportTextEncoding,
|
||||
TableImportColumnMapping,
|
||||
TableImportParseOptions,
|
||||
TableImportPreviewRequest,
|
||||
|
|
|
|||
|
|
@ -2099,6 +2099,7 @@ export type TableImportMode = "append" | "truncate";
|
|||
export type TableImportStatus = "running" | "done" | "error" | "cancelled";
|
||||
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel";
|
||||
export type TableImportJsonShape = "auto" | "objects" | "arrays";
|
||||
export type TableImportTextEncoding = "auto" | "utf8" | "gbk" | "utf16Le" | "utf16Be";
|
||||
|
||||
export interface TableImportColumnMapping {
|
||||
sourceColumn: string;
|
||||
|
|
@ -2108,6 +2109,7 @@ export interface TableImportColumnMapping {
|
|||
|
||||
export interface TableImportParseOptions {
|
||||
delimiter?: string | null;
|
||||
encoding?: TableImportTextEncoding | null;
|
||||
hasHeader?: boolean | null;
|
||||
titleRow?: number | null;
|
||||
dataStartRow?: number | null;
|
||||
|
|
@ -2136,6 +2138,7 @@ export interface TableImportPreview {
|
|||
columns: string[];
|
||||
rows: unknown[][];
|
||||
totalRows: number;
|
||||
effectiveEncoding?: TableImportTextEncoding | null;
|
||||
sheets?: string[];
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::io::{Read as IoRead, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
|
||||
use calamine::{open_workbook_auto, Data, ExcelDateTime, Reader as CalamineReader};
|
||||
|
|
@ -29,6 +29,7 @@ pub struct ParsedImportFile {
|
|||
pub columns: Vec<String>,
|
||||
pub rows: Vec<Vec<serde_json::Value>>,
|
||||
pub total_rows: usize,
|
||||
pub effective_encoding: Option<TableImportTextEncoding>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
|
|
@ -99,10 +100,43 @@ pub enum TableImportJsonShape {
|
|||
Arrays,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum TableImportTextEncoding {
|
||||
Auto,
|
||||
Utf8,
|
||||
Gbk,
|
||||
Utf16Le,
|
||||
Utf16Be,
|
||||
}
|
||||
|
||||
impl TableImportTextEncoding {
|
||||
fn encoding(self) -> Option<&'static encoding_rs::Encoding> {
|
||||
match self {
|
||||
TableImportTextEncoding::Auto => None,
|
||||
TableImportTextEncoding::Utf8 => Some(encoding_rs::UTF_8),
|
||||
TableImportTextEncoding::Gbk => Some(encoding_rs::GBK),
|
||||
TableImportTextEncoding::Utf16Le => Some(encoding_rs::UTF_16LE),
|
||||
TableImportTextEncoding::Utf16Be => Some(encoding_rs::UTF_16BE),
|
||||
}
|
||||
}
|
||||
|
||||
fn label(self) -> &'static str {
|
||||
match self {
|
||||
TableImportTextEncoding::Auto => "auto",
|
||||
TableImportTextEncoding::Utf8 => "UTF-8",
|
||||
TableImportTextEncoding::Gbk => "GBK / GB18030",
|
||||
TableImportTextEncoding::Utf16Le => "UTF-16 LE",
|
||||
TableImportTextEncoding::Utf16Be => "UTF-16 BE",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TableImportParseOptions {
|
||||
pub delimiter: Option<String>,
|
||||
pub encoding: Option<TableImportTextEncoding>,
|
||||
pub has_header: Option<bool>,
|
||||
pub title_row: Option<usize>,
|
||||
pub data_start_row: Option<usize>,
|
||||
|
|
@ -118,6 +152,7 @@ impl Default for TableImportParseOptions {
|
|||
fn default() -> Self {
|
||||
Self {
|
||||
delimiter: None,
|
||||
encoding: Some(TableImportTextEncoding::Auto),
|
||||
has_header: None,
|
||||
title_row: None,
|
||||
data_start_row: None,
|
||||
|
|
@ -179,6 +214,8 @@ pub struct TableImportPreview {
|
|||
pub columns: Vec<String>,
|
||||
pub rows: Vec<Vec<serde_json::Value>>,
|
||||
pub total_rows: usize,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub effective_encoding: Option<TableImportTextEncoding>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub sheets: Vec<String>,
|
||||
}
|
||||
|
|
@ -361,14 +398,240 @@ pub fn csv_value(value: &str) -> serde_json::Value {
|
|||
)
|
||||
}
|
||||
|
||||
const IMPORT_ENCODING_READ_CHUNK_BYTES: usize = 16 * 1024;
|
||||
|
||||
struct StrictTranscodingReader<R> {
|
||||
reader: R,
|
||||
decoder: encoding_rs::Decoder,
|
||||
encoding: TableImportTextEncoding,
|
||||
pending_input: Vec<u8>,
|
||||
pending_output: Vec<u8>,
|
||||
output_offset: usize,
|
||||
reached_eof: bool,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl<R: IoRead> StrictTranscodingReader<R> {
|
||||
fn new(reader: R, encoding: TableImportTextEncoding) -> Result<Self, String> {
|
||||
let decoder = encoding
|
||||
.encoding()
|
||||
.ok_or_else(|| "Automatic text encoding must be resolved before decoding".to_string())?
|
||||
.new_decoder_without_bom_handling();
|
||||
Ok(Self {
|
||||
reader,
|
||||
decoder,
|
||||
encoding,
|
||||
pending_input: Vec::with_capacity(IMPORT_ENCODING_READ_CHUNK_BYTES),
|
||||
pending_output: Vec::new(),
|
||||
output_offset: 0,
|
||||
reached_eof: false,
|
||||
finished: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn invalid_data_error(&self) -> std::io::Error {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
format!("Invalid byte sequence for {} encoding", self.encoding.label()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: IoRead> IoRead for StrictTranscodingReader<R> {
|
||||
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
|
||||
if buffer.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
loop {
|
||||
if self.output_offset < self.pending_output.len() {
|
||||
let available = &self.pending_output[self.output_offset..];
|
||||
let copied = available.len().min(buffer.len());
|
||||
buffer[..copied].copy_from_slice(&available[..copied]);
|
||||
self.output_offset += copied;
|
||||
if self.output_offset == self.pending_output.len() {
|
||||
self.pending_output.clear();
|
||||
self.output_offset = 0;
|
||||
}
|
||||
return Ok(copied);
|
||||
}
|
||||
if self.finished {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
if self.pending_input.is_empty() && !self.reached_eof {
|
||||
let mut input = [0u8; IMPORT_ENCODING_READ_CHUNK_BYTES];
|
||||
let read = self.reader.read(&mut input)?;
|
||||
if read == 0 {
|
||||
self.reached_eof = true;
|
||||
} else {
|
||||
self.pending_input.extend_from_slice(&input[..read]);
|
||||
}
|
||||
}
|
||||
|
||||
let output_capacity = self
|
||||
.decoder
|
||||
.max_utf8_buffer_length_without_replacement(self.pending_input.len())
|
||||
.unwrap_or(self.pending_input.len().saturating_mul(3).saturating_add(4))
|
||||
.max(4);
|
||||
let mut output = vec![0u8; output_capacity];
|
||||
let (result, read, written) =
|
||||
self.decoder.decode_to_utf8_without_replacement(&self.pending_input, &mut output, self.reached_eof);
|
||||
self.pending_input.drain(..read);
|
||||
output.truncate(written);
|
||||
self.pending_output = output;
|
||||
|
||||
match result {
|
||||
encoding_rs::DecoderResult::Malformed(_, _) => return Err(self.invalid_data_error()),
|
||||
encoding_rs::DecoderResult::InputEmpty if self.reached_eof => self.finished = true,
|
||||
encoding_rs::DecoderResult::InputEmpty | encoding_rs::DecoderResult::OutputFull => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn bom_text_encoding(bytes: &[u8]) -> Option<(TableImportTextEncoding, usize)> {
|
||||
if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
|
||||
Some((TableImportTextEncoding::Utf8, 3))
|
||||
} else if bytes.starts_with(&[0xFF, 0xFE]) {
|
||||
Some((TableImportTextEncoding::Utf16Le, 2))
|
||||
} else if bytes.starts_with(&[0xFE, 0xFF]) {
|
||||
Some((TableImportTextEncoding::Utf16Be, 2))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn matching_bom_len(bytes: &[u8], encoding: TableImportTextEncoding) -> usize {
|
||||
bom_text_encoding(bytes).filter(|(bom_encoding, _)| *bom_encoding == encoding).map(|(_, len)| len).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn reader_is_valid_for_encoding<R: IoRead>(reader: R, encoding: TableImportTextEncoding) -> Result<bool, String> {
|
||||
let mut reader = StrictTranscodingReader::new(reader, encoding)?;
|
||||
match std::io::copy(&mut reader, &mut std::io::sink()) {
|
||||
Ok(_) => Ok(true),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::InvalidData => Ok(false),
|
||||
Err(error) => Err(error.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn auto_detect_text_encoding_from_bytes(bytes: &[u8]) -> Result<(TableImportTextEncoding, usize), String> {
|
||||
if let Some(detected) = bom_text_encoding(bytes) {
|
||||
return Ok(detected);
|
||||
}
|
||||
for encoding in [TableImportTextEncoding::Utf8, TableImportTextEncoding::Gbk] {
|
||||
if reader_is_valid_for_encoding(std::io::Cursor::new(bytes), encoding)? {
|
||||
return Ok((encoding, 0));
|
||||
}
|
||||
}
|
||||
Err("Could not detect text encoding; select UTF-8, GBK / GB18030, or UTF-16 manually".to_string())
|
||||
}
|
||||
|
||||
fn resolve_text_encoding_from_bytes(
|
||||
bytes: &[u8],
|
||||
requested: Option<TableImportTextEncoding>,
|
||||
) -> Result<(TableImportTextEncoding, usize), String> {
|
||||
let requested = requested.unwrap_or(TableImportTextEncoding::Auto);
|
||||
if requested == TableImportTextEncoding::Auto {
|
||||
auto_detect_text_encoding_from_bytes(bytes)
|
||||
} else {
|
||||
Ok((requested, matching_bom_len(bytes, requested)))
|
||||
}
|
||||
}
|
||||
|
||||
fn auto_detect_text_encoding_from_file(path: &str) -> Result<(TableImportTextEncoding, usize), String> {
|
||||
let mut file = File::open(path).map_err(|error| error.to_string())?;
|
||||
let mut prefix = [0u8; 3];
|
||||
let prefix_len = file.read(&mut prefix).map_err(|error| error.to_string())?;
|
||||
if let Some(detected) = bom_text_encoding(&prefix[..prefix_len]) {
|
||||
return Ok(detected);
|
||||
}
|
||||
|
||||
for encoding in [TableImportTextEncoding::Utf8, TableImportTextEncoding::Gbk] {
|
||||
let file = File::open(path).map_err(|error| error.to_string())?;
|
||||
if reader_is_valid_for_encoding(file, encoding)? {
|
||||
return Ok((encoding, 0));
|
||||
}
|
||||
}
|
||||
Err("Could not detect text encoding; select UTF-8, GBK / GB18030, or UTF-16 manually".to_string())
|
||||
}
|
||||
|
||||
fn resolve_text_encoding_from_file(
|
||||
path: &str,
|
||||
requested: Option<TableImportTextEncoding>,
|
||||
) -> Result<(TableImportTextEncoding, usize), String> {
|
||||
let requested = requested.unwrap_or(TableImportTextEncoding::Auto);
|
||||
if requested == TableImportTextEncoding::Auto {
|
||||
return auto_detect_text_encoding_from_file(path);
|
||||
}
|
||||
|
||||
let mut file = File::open(path).map_err(|error| error.to_string())?;
|
||||
let mut prefix = [0u8; 3];
|
||||
let prefix_len = file.read(&mut prefix).map_err(|error| error.to_string())?;
|
||||
Ok((requested, matching_bom_len(&prefix[..prefix_len], requested)))
|
||||
}
|
||||
|
||||
fn open_delimited_csv_reader(
|
||||
path: &str,
|
||||
source_format: TableImportSourceFormat,
|
||||
options: &TableImportParseOptions,
|
||||
) -> Result<(csv::Reader<StrictTranscodingReader<File>>, DelimitedParseConfig, TableImportTextEncoding), String> {
|
||||
let config = effective_delimited_config(source_format, options)?;
|
||||
let (encoding, bom_len) = resolve_text_encoding_from_file(path, options.encoding)?;
|
||||
let mut file = File::open(path).map_err(|error| error.to_string())?;
|
||||
file.seek(SeekFrom::Start(bom_len as u64)).map_err(|error| error.to_string())?;
|
||||
let transcoded = StrictTranscodingReader::new(file, encoding)?;
|
||||
let reader =
|
||||
csv::ReaderBuilder::new().delimiter(config.delimiter).has_headers(false).flexible(true).from_reader(transcoded);
|
||||
Ok((reader, config, encoding))
|
||||
}
|
||||
|
||||
pub fn parse_delimited_reader<R: std::io::Read>(
|
||||
reader: R,
|
||||
config: DelimitedParseConfig,
|
||||
preview_limit: usize,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
let mut reader =
|
||||
csv::ReaderBuilder::new().delimiter(config.delimiter).has_headers(false).flexible(true).from_reader(reader);
|
||||
parse_decoded_delimited_reader(reader, config, preview_limit, TableImportTextEncoding::Utf8)
|
||||
}
|
||||
|
||||
fn parse_decoded_delimited_reader<R: IoRead>(
|
||||
reader: R,
|
||||
config: DelimitedParseConfig,
|
||||
preview_limit: usize,
|
||||
effective_encoding: TableImportTextEncoding,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
let reader =
|
||||
csv::ReaderBuilder::new().delimiter(config.delimiter).has_headers(false).flexible(true).from_reader(reader);
|
||||
parse_csv_reader(reader, config, preview_limit, effective_encoding)
|
||||
}
|
||||
|
||||
pub fn parse_delimited_bytes_with_options(
|
||||
bytes: &[u8],
|
||||
source_format: TableImportSourceFormat,
|
||||
options: &TableImportParseOptions,
|
||||
preview_limit: usize,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
let (encoding, bom_len) = resolve_text_encoding_from_bytes(bytes, options.encoding)?;
|
||||
let reader = StrictTranscodingReader::new(std::io::Cursor::new(&bytes[bom_len..]), encoding)?;
|
||||
parse_decoded_delimited_reader(reader, effective_delimited_config(source_format, options)?, preview_limit, encoding)
|
||||
}
|
||||
|
||||
pub fn parse_delimited_file_with_options(
|
||||
path: &str,
|
||||
source_format: TableImportSourceFormat,
|
||||
options: &TableImportParseOptions,
|
||||
preview_limit: usize,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
let (reader, config, encoding) = open_delimited_csv_reader(path, source_format, options)?;
|
||||
parse_csv_reader(reader, config, preview_limit, encoding)
|
||||
}
|
||||
|
||||
fn parse_csv_reader<R: IoRead>(
|
||||
mut reader: csv::Reader<R>,
|
||||
config: DelimitedParseConfig,
|
||||
preview_limit: usize,
|
||||
effective_encoding: TableImportTextEncoding,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
let mut rows = Vec::new();
|
||||
let mut total_rows = 0;
|
||||
let mut columns = Vec::new();
|
||||
|
|
@ -396,13 +659,7 @@ pub fn parse_delimited_reader<R: std::io::Read>(
|
|||
if rows.len() >= preview_limit {
|
||||
continue;
|
||||
}
|
||||
let mut row = Vec::with_capacity(columns.len());
|
||||
for index in 0..columns.len() {
|
||||
row.push(
|
||||
record.get(index).map(|value| csv_value_with_config(value, config)).unwrap_or(serde_json::Value::Null),
|
||||
);
|
||||
}
|
||||
rows.push(row);
|
||||
rows.push(delimited_record_to_row(&record, columns.len(), config));
|
||||
}
|
||||
if columns.is_empty() {
|
||||
return Err("Import file has no columns in the selected row range".to_string());
|
||||
|
|
@ -410,30 +667,7 @@ pub fn parse_delimited_reader<R: std::io::Read>(
|
|||
if total_rows == 0 {
|
||||
return Err("Import file has no data rows in the selected row range".to_string());
|
||||
}
|
||||
Ok(ParsedImportFile { columns, rows, total_rows })
|
||||
}
|
||||
|
||||
pub fn parse_delimited_bytes_with_options(
|
||||
bytes: &[u8],
|
||||
source_format: TableImportSourceFormat,
|
||||
options: &TableImportParseOptions,
|
||||
preview_limit: usize,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
parse_delimited_reader(
|
||||
bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(bytes),
|
||||
effective_delimited_config(source_format, options)?,
|
||||
preview_limit,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn parse_delimited_file_with_options(
|
||||
path: &str,
|
||||
source_format: TableImportSourceFormat,
|
||||
options: &TableImportParseOptions,
|
||||
preview_limit: usize,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
let file = File::open(path).map_err(|e| e.to_string())?;
|
||||
parse_delimited_reader(file, effective_delimited_config(source_format, options)?, preview_limit)
|
||||
Ok(ParsedImportFile { columns, rows, total_rows, effective_encoding: Some(effective_encoding) })
|
||||
}
|
||||
|
||||
pub fn parse_csv_bytes(bytes: &[u8], preview_limit: usize) -> Result<ParsedImportFile, String> {
|
||||
|
|
@ -505,7 +739,7 @@ pub fn parse_json_bytes_with_options(
|
|||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
return Ok(ParsedImportFile { columns, rows, total_rows: items.len() });
|
||||
return Ok(ParsedImportFile { columns, rows, total_rows: items.len(), effective_encoding: None });
|
||||
}
|
||||
|
||||
if all_arrays {
|
||||
|
|
@ -524,7 +758,7 @@ pub fn parse_json_bytes_with_options(
|
|||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
return Ok(ParsedImportFile { columns, rows, total_rows: items.len() });
|
||||
return Ok(ParsedImportFile { columns, rows, total_rows: items.len(), effective_encoding: None });
|
||||
}
|
||||
|
||||
Err("JSON rows must all be objects or all be arrays; mixed row shapes are not supported".to_string())
|
||||
|
|
@ -992,7 +1226,7 @@ pub fn parse_xlsx_file_with_options(
|
|||
if total_rows == 0 {
|
||||
return Err("Import file has no data rows in the selected row range".to_string());
|
||||
}
|
||||
Ok(ParsedImportFile { columns, rows, total_rows })
|
||||
Ok(ParsedImportFile { columns, rows, total_rows, effective_encoding: None })
|
||||
}
|
||||
|
||||
pub fn parse_xlsx_file(path: &str, preview_limit: usize) -> Result<ParsedImportFile, String> {
|
||||
|
|
@ -1598,6 +1832,7 @@ pub async fn preview_table_import_file_with_request(
|
|||
columns: parsed.columns,
|
||||
rows: parsed.rows,
|
||||
total_rows: parsed.total_rows,
|
||||
effective_encoding: parsed.effective_encoding,
|
||||
sheets,
|
||||
})
|
||||
}
|
||||
|
|
@ -1742,24 +1977,17 @@ where
|
|||
}
|
||||
}
|
||||
|
||||
let config = match effective_delimited_config(source_format, &request.parse_options) {
|
||||
Ok(config) => config,
|
||||
Err(error) => return Err(emit_import_error(&mut progress_callback, request, 0, total_rows, error)),
|
||||
};
|
||||
let file = match File::open(&request.file_path) {
|
||||
Ok(file) => file,
|
||||
Err(error) => {
|
||||
return Err(emit_import_error(
|
||||
&mut progress_callback,
|
||||
request,
|
||||
0,
|
||||
total_rows,
|
||||
format!("Import source is no longer available: {error}"),
|
||||
));
|
||||
}
|
||||
};
|
||||
let mut reader =
|
||||
csv::ReaderBuilder::new().delimiter(config.delimiter).has_headers(false).flexible(true).from_reader(file);
|
||||
let mut streaming_options = request.parse_options.clone();
|
||||
if streaming_options.encoding.unwrap_or(TableImportTextEncoding::Auto) == TableImportTextEncoding::Auto {
|
||||
streaming_options.encoding = parsed.effective_encoding;
|
||||
}
|
||||
let (mut reader, config, _) =
|
||||
match open_delimited_csv_reader(&request.file_path, source_format, &streaming_options) {
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
return Err(emit_import_error(&mut progress_callback, request, 0, total_rows, error));
|
||||
}
|
||||
};
|
||||
let (columns, first_record) = match delimited_columns_and_first_record(&mut reader, config) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return Err(emit_import_error(&mut progress_callback, request, 0, total_rows, error)),
|
||||
|
|
@ -2122,6 +2350,152 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_detects_and_parses_gbk_csv() {
|
||||
let (bytes, _, had_errors) = encoding_rs::GBK.encode("id,name\n1,中文\n2,上海\n");
|
||||
assert!(!had_errors);
|
||||
|
||||
let parsed = parse_delimited_bytes_with_options(
|
||||
bytes.as_ref(),
|
||||
TableImportSourceFormat::Csv,
|
||||
&TableImportParseOptions::default(),
|
||||
10,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(parsed.effective_encoding, Some(TableImportTextEncoding::Gbk));
|
||||
assert_eq!(parsed.columns, vec!["id", "name"]);
|
||||
assert_eq!(parsed.rows[0], vec![serde_json::json!("1"), serde_json::json!("中文")]);
|
||||
assert_eq!(parsed.rows[1], vec![serde_json::json!("2"), serde_json::json!("上海")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_utf8_rejects_gbk_csv_without_replacing_data() {
|
||||
let (bytes, _, had_errors) = encoding_rs::GBK.encode("id,name\n1,中文\n");
|
||||
assert!(!had_errors);
|
||||
let options = TableImportParseOptions {
|
||||
encoding: Some(TableImportTextEncoding::Utf8),
|
||||
..TableImportParseOptions::default()
|
||||
};
|
||||
|
||||
let error =
|
||||
parse_delimited_bytes_with_options(bytes.as_ref(), TableImportSourceFormat::Csv, &options, 10).unwrap_err();
|
||||
|
||||
assert!(error.contains("Invalid byte sequence for UTF-8 encoding"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gbk_option_decodes_gb18030_four_byte_characters() {
|
||||
let (bytes, _, had_errors) = encoding_rs::GB18030.encode("id,name\n1,😀\n");
|
||||
assert!(!had_errors);
|
||||
|
||||
let parsed = parse_delimited_bytes_with_options(
|
||||
bytes.as_ref(),
|
||||
TableImportSourceFormat::Csv,
|
||||
&TableImportParseOptions::default(),
|
||||
10,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(parsed.effective_encoding, Some(TableImportTextEncoding::Gbk));
|
||||
assert_eq!(parsed.rows[0], vec![serde_json::json!("1"), serde_json::json!("😀")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_detects_utf16le_bom_csv() {
|
||||
let mut bytes = vec![0xFF, 0xFE];
|
||||
for unit in "id,name\n1,中文\n".encode_utf16() {
|
||||
bytes.extend_from_slice(&unit.to_le_bytes());
|
||||
}
|
||||
|
||||
let parsed = parse_delimited_bytes_with_options(
|
||||
&bytes,
|
||||
TableImportSourceFormat::Csv,
|
||||
&TableImportParseOptions::default(),
|
||||
10,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(parsed.effective_encoding, Some(TableImportTextEncoding::Utf16Le));
|
||||
assert_eq!(parsed.rows[0], vec![serde_json::json!("1"), serde_json::json!("中文")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_detects_utf16be_bom_csv() {
|
||||
let mut bytes = vec![0xFE, 0xFF];
|
||||
for unit in "id,name\n1,中文\n".encode_utf16() {
|
||||
bytes.extend_from_slice(&unit.to_be_bytes());
|
||||
}
|
||||
|
||||
let parsed = parse_delimited_bytes_with_options(
|
||||
&bytes,
|
||||
TableImportSourceFormat::Csv,
|
||||
&TableImportParseOptions::default(),
|
||||
10,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(parsed.effective_encoding, Some(TableImportTextEncoding::Utf16Be));
|
||||
assert_eq!(parsed.columns, vec!["id", "name"]);
|
||||
assert_eq!(parsed.rows[0], vec![serde_json::json!("1"), serde_json::json!("中文")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_utf16le_parses_csv_without_bom() {
|
||||
let bytes = "id,name\n1,中文\n".encode_utf16().flat_map(u16::to_le_bytes).collect::<Vec<_>>();
|
||||
let options = TableImportParseOptions {
|
||||
encoding: Some(TableImportTextEncoding::Utf16Le),
|
||||
..TableImportParseOptions::default()
|
||||
};
|
||||
|
||||
let parsed = parse_delimited_bytes_with_options(&bytes, TableImportSourceFormat::Csv, &options, 10).unwrap();
|
||||
|
||||
assert_eq!(parsed.effective_encoding, Some(TableImportTextEncoding::Utf16Le));
|
||||
assert_eq!(parsed.columns, vec!["id", "name"]);
|
||||
assert_eq!(parsed.rows[0], vec![serde_json::json!("1"), serde_json::json!("中文")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gbk_decoder_preserves_multibyte_character_across_read_chunks() {
|
||||
let ascii_prefix = "a".repeat(IMPORT_ENCODING_READ_CHUNK_BYTES - "name\n".len() - 1);
|
||||
let csv = format!("name\n{ascii_prefix}中\n");
|
||||
let (bytes, _, had_errors) = encoding_rs::GBK.encode(&csv);
|
||||
assert!(!had_errors);
|
||||
let options = TableImportParseOptions {
|
||||
encoding: Some(TableImportTextEncoding::Gbk),
|
||||
..TableImportParseOptions::default()
|
||||
};
|
||||
|
||||
let parsed =
|
||||
parse_delimited_bytes_with_options(bytes.as_ref(), TableImportSourceFormat::Csv, &options, 10).unwrap();
|
||||
|
||||
assert_eq!(parsed.rows[0][0], serde_json::json!(format!("{ascii_prefix}中")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preview_reads_real_gbk_file_and_reports_detected_encoding() {
|
||||
let path = std::env::temp_dir().join(format!("dbx-table-import-gbk-{}.csv", uuid::Uuid::new_v4()));
|
||||
let (bytes, _, had_errors) = encoding_rs::GBK.encode("编号,城市\n1,北京\n2,上海\n");
|
||||
assert!(!had_errors);
|
||||
std::fs::write(&path, bytes.as_ref()).unwrap();
|
||||
|
||||
let preview = preview_table_import_file_with_request(TableImportPreviewRequest {
|
||||
file_path: path.to_string_lossy().to_string(),
|
||||
source_ref: None,
|
||||
source_format: Some(TableImportSourceFormat::Csv),
|
||||
parse_options: TableImportParseOptions::default(),
|
||||
preview_limit: Some(10),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let _ = std::fs::remove_file(path);
|
||||
|
||||
assert_eq!(preview.effective_encoding, Some(TableImportTextEncoding::Gbk));
|
||||
assert_eq!(preview.columns, vec!["编号", "城市"]);
|
||||
assert_eq!(preview.total_rows, 2);
|
||||
assert_eq!(preview.rows[0], vec![serde_json::json!("1"), serde_json::json!("北京")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_tsv_with_tab_delimiter() {
|
||||
let parsed = parse_delimited_bytes(b"id\tname\n1\tAda\n", b'\t', 10).unwrap();
|
||||
|
|
@ -2385,6 +2759,7 @@ mod tests {
|
|||
],
|
||||
],
|
||||
total_rows: 2,
|
||||
effective_encoding: None,
|
||||
};
|
||||
let mappings = data
|
||||
.columns
|
||||
|
|
@ -2418,8 +2793,12 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn create_table_plan_requires_target_table_name() {
|
||||
let data =
|
||||
ParsedImportFile { columns: vec!["id".to_string()], rows: vec![vec![serde_json::json!(1)]], total_rows: 1 };
|
||||
let data = ParsedImportFile {
|
||||
columns: vec!["id".to_string()],
|
||||
rows: vec![vec![serde_json::json!(1)]],
|
||||
total_rows: 1,
|
||||
effective_encoding: None,
|
||||
};
|
||||
let mappings = vec![TableImportColumnMapping {
|
||||
source_column: "id".to_string(),
|
||||
target_column: "id".to_string(),
|
||||
|
|
@ -2437,6 +2816,7 @@ mod tests {
|
|||
columns: vec!["notes".to_string()],
|
||||
rows: vec![vec![serde_json::json!("long text")]],
|
||||
total_rows: 1,
|
||||
effective_encoding: None,
|
||||
};
|
||||
let mappings = vec![TableImportColumnMapping {
|
||||
source_column: "notes".to_string(),
|
||||
|
|
@ -2455,6 +2835,7 @@ mod tests {
|
|||
columns: vec!["code".to_string(), "amount".to_string()],
|
||||
rows: vec![vec![serde_json::json!("1001"), serde_json::json!("12.5")]],
|
||||
total_rows: 1,
|
||||
effective_encoding: None,
|
||||
};
|
||||
let mappings = vec![
|
||||
TableImportColumnMapping {
|
||||
|
|
@ -2487,6 +2868,7 @@ mod tests {
|
|||
columns: vec!["name".to_string()],
|
||||
rows: vec![vec![serde_json::json!("Ada")]],
|
||||
total_rows: 1,
|
||||
effective_encoding: None,
|
||||
};
|
||||
let mappings = vec![TableImportColumnMapping {
|
||||
source_column: "name".to_string(),
|
||||
|
|
@ -2521,6 +2903,7 @@ mod tests {
|
|||
vec![serde_json::json!(3), serde_json::Value::Null, serde_json::json!("z")],
|
||||
],
|
||||
total_rows: 3,
|
||||
effective_encoding: None,
|
||||
};
|
||||
|
||||
let batches =
|
||||
|
|
@ -2549,6 +2932,7 @@ mod tests {
|
|||
columns: vec!["id".to_string()],
|
||||
rows: vec![vec![serde_json::json!(1)], vec![serde_json::json!(2)]],
|
||||
total_rows: 2,
|
||||
effective_encoding: None,
|
||||
};
|
||||
|
||||
let batches =
|
||||
|
|
@ -2662,6 +3046,7 @@ mod tests {
|
|||
vec![serde_json::json!(3), serde_json::Value::Null],
|
||||
],
|
||||
total_rows: 3,
|
||||
effective_encoding: None,
|
||||
};
|
||||
|
||||
let batches =
|
||||
|
|
@ -2707,6 +3092,7 @@ mod tests {
|
|||
serde_json::json!("2026-05-12T00:00:00+00:00"),
|
||||
]],
|
||||
total_rows: 1,
|
||||
effective_encoding: None,
|
||||
};
|
||||
|
||||
let batches = build_import_insert_batches(
|
||||
|
|
@ -2740,6 +3126,7 @@ mod tests {
|
|||
columns: vec!["name".to_string()],
|
||||
rows: vec![vec![serde_json::json!("Tiếng Việt")]],
|
||||
total_rows: 1,
|
||||
effective_encoding: None,
|
||||
};
|
||||
|
||||
let batches = build_import_insert_batches(
|
||||
|
|
|
|||
Loading…
Reference in New Issue