Merge pull request #59 from SuLea-IT/codex/table-file-import

feat: import table data from files
This commit is contained in:
skyler 2026-05-02 19:26:19 +08:00 committed by GitHub
commit b335aacdba
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 1448 additions and 9 deletions

98
src-tauri/Cargo.lock generated
View File

@ -529,6 +529,15 @@ dependencies = [
"num-traits",
]
[[package]]
name = "atoi_simd"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a49e05797ca52e312a0c658938b7d00693ef037799ef7187678f212d7684cf"
dependencies = [
"debug_unsafe",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
@ -872,6 +881,23 @@ dependencies = [
"system-deps",
]
[[package]]
name = "calamine"
version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1a9acfeb1555aa8def91fe8ff208aadaea850c109968ec35ac965edbe7d210b"
dependencies = [
"atoi_simd",
"byteorder",
"codepage",
"encoding_rs",
"fast-float2",
"log",
"quick-xml 0.37.5",
"serde",
"zip 4.6.1",
]
[[package]]
name = "camino"
version = "1.2.2"
@ -1061,6 +1087,15 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746"
[[package]]
name = "codepage"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4"
dependencies = [
"encoding_rs",
]
[[package]]
name = "combine"
version = "4.6.7"
@ -1412,6 +1447,27 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "csv"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
dependencies = [
"csv-core",
"itoa",
"ryu",
"serde_core",
]
[[package]]
name = "csv-core"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
dependencies = [
"memchr",
]
[[package]]
name = "ctor"
version = "0.2.9"
@ -1551,7 +1607,9 @@ name = "dbx"
version = "0.3.6"
dependencies = [
"anyhow",
"calamine",
"chrono",
"csv",
"duckdb",
"futures",
"keyring",
@ -1580,6 +1638,12 @@ dependencies = [
"uuid",
]
[[package]]
name = "debug_unsafe"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2"
[[package]]
name = "delegate"
version = "0.13.5"
@ -2092,6 +2156,12 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fast-float2"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
[[package]]
name = "fastrand"
version = "2.4.1"
@ -3669,7 +3739,7 @@ dependencies = [
"serde_json",
"tar",
"vcpkg",
"zip",
"zip 6.0.0",
]
[[package]]
@ -5004,7 +5074,7 @@ checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1"
dependencies = [
"base64 0.22.1",
"indexmap 2.14.0",
"quick-xml",
"quick-xml 0.39.2",
"serde",
"time",
]
@ -5242,6 +5312,16 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "quick-xml"
version = "0.37.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb"
dependencies = [
"encoding_rs",
"memchr",
]
[[package]]
name = "quick-xml"
version = "0.39.2"
@ -9581,6 +9661,20 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "zip"
version = "4.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1"
dependencies = [
"arbitrary",
"crc32fast",
"flate2",
"indexmap 2.14.0",
"memchr",
"zopfli",
]
[[package]]
name = "zip"
version = "6.0.0"

View File

@ -49,3 +49,5 @@ oracle-rs = "0.1"
percent-encoding = "2"
russh = "0.60"
keyring = { version = "3.6.3", features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] }
csv = "1.4.0"
calamine = "0.30.1"

View File

@ -8,5 +8,6 @@ pub mod query_cancel;
pub mod redis_cmd;
pub mod schema;
pub mod sql_file;
pub mod table_import;
pub mod transfer;
pub mod update;

View File

@ -0,0 +1,770 @@
use std::collections::HashSet;
use std::path::Path;
use std::sync::Arc;
use calamine::{open_workbook_auto, Data, Reader};
use serde::{Deserialize, Serialize};
use tauri::{AppHandle, Emitter, State};
use tokio::sync::RwLock;
use crate::commands::connection::AppState;
use crate::commands::transfer::{execute_on_pool, generate_insert, get_db_type, qualified_table};
use crate::models::connection::DatabaseType;
const DEFAULT_PREVIEW_LIMIT: usize = 50;
const DEFAULT_BATCH_SIZE: usize = 500;
static CANCELLED_IMPORTS: std::sync::LazyLock<RwLock<HashSet<String>>> =
std::sync::LazyLock::new(|| RwLock::new(HashSet::new()));
#[derive(Debug, Clone)]
struct ParsedImportFile {
columns: Vec<String>,
rows: Vec<Vec<serde_json::Value>>,
total_rows: usize,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct ImportSqlBatch {
sql: String,
row_count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TableImportColumnMapping {
pub source_column: String,
pub target_column: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum TableImportMode {
Append,
Truncate,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TableImportRequest {
pub import_id: String,
pub connection_id: String,
pub database: String,
pub schema: String,
pub table: String,
pub file_path: String,
pub mappings: Vec<TableImportColumnMapping>,
pub mode: TableImportMode,
pub batch_size: usize,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TableImportPreview {
pub file_name: String,
pub file_path: String,
pub file_type: String,
pub size_bytes: u64,
pub columns: Vec<String>,
pub rows: Vec<Vec<serde_json::Value>>,
pub total_rows: usize,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TableImportSummary {
pub import_id: String,
pub rows_imported: usize,
pub total_rows: usize,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TableImportProgress {
pub import_id: String,
pub status: TableImportStatus,
pub rows_imported: usize,
pub total_rows: usize,
pub error: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum TableImportStatus {
Running,
Done,
Error,
Cancelled,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ImportFileKind {
Csv,
Tsv,
Json,
Xlsx,
}
impl ImportFileKind {
fn label(self) -> &'static str {
match self {
ImportFileKind::Csv => "csv",
ImportFileKind::Tsv => "tsv",
ImportFileKind::Json => "json",
ImportFileKind::Xlsx => "xlsx",
}
}
}
fn import_file_kind(path: &str) -> Result<ImportFileKind, String> {
let lower = path.to_lowercase();
if lower.ends_with(".csv") {
Ok(ImportFileKind::Csv)
} else if lower.ends_with(".tsv") {
Ok(ImportFileKind::Tsv)
} else if lower.ends_with(".json") {
Ok(ImportFileKind::Json)
} else if lower.ends_with(".xlsx") || lower.ends_with(".xlsm") || lower.ends_with(".xls") {
Ok(ImportFileKind::Xlsx)
} else {
Err("Unsupported import file type".to_string())
}
}
fn normalize_header(value: &str, index: usize) -> String {
let trimmed = value.trim();
if trimmed.is_empty() {
format!("column_{}", index + 1)
} else {
trimmed.to_string()
}
}
fn csv_value(value: &str) -> serde_json::Value {
if value.is_empty() {
serde_json::Value::Null
} else {
serde_json::Value::String(value.to_string())
}
}
fn parse_delimited_bytes(
bytes: &[u8],
delimiter: u8,
preview_limit: usize,
) -> Result<ParsedImportFile, String> {
let mut reader = csv::ReaderBuilder::new()
.delimiter(delimiter)
.flexible(true)
.from_reader(bytes);
let columns = reader
.headers()
.map_err(|e| e.to_string())?
.iter()
.enumerate()
.map(|(index, header)| normalize_header(header, index))
.collect::<Vec<_>>();
if columns.is_empty() {
return Err("Import file has no columns".to_string());
}
let mut rows = Vec::new();
let mut total_rows = 0;
for record in reader.records() {
let record = record.map_err(|e| e.to_string())?;
total_rows += 1;
if rows.len() >= preview_limit {
continue;
}
let mut row = Vec::with_capacity(columns.len());
for index in 0..columns.len() {
row.push(
record
.get(index)
.map(csv_value)
.unwrap_or(serde_json::Value::Null),
);
}
rows.push(row);
}
Ok(ParsedImportFile {
columns,
rows,
total_rows,
})
}
fn parse_csv_bytes(bytes: &[u8], preview_limit: usize) -> Result<ParsedImportFile, String> {
parse_delimited_bytes(bytes, b',', preview_limit)
}
fn parse_json_bytes(bytes: &[u8], preview_limit: usize) -> Result<ParsedImportFile, String> {
let value: serde_json::Value = serde_json::from_slice(bytes).map_err(|e| e.to_string())?;
let items = match value {
serde_json::Value::Array(items) => items,
serde_json::Value::Object(_) => vec![value],
_ => return Err("JSON import must be an object or an array".to_string()),
};
if items.is_empty() {
return Err("Import file has no rows".to_string());
}
if items.iter().all(|item| item.is_object()) {
let mut columns = Vec::new();
for item in &items {
if let Some(obj) = item.as_object() {
for key in obj.keys() {
if !columns.contains(key) {
columns.push(key.clone());
}
}
}
}
if columns.is_empty() {
return Err("Import file has no columns".to_string());
}
let rows = items
.iter()
.take(preview_limit)
.map(|item| {
let obj = item.as_object().expect("checked object JSON row");
columns
.iter()
.map(|column| obj.get(column).cloned().unwrap_or(serde_json::Value::Null))
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
return Ok(ParsedImportFile {
columns,
rows,
total_rows: items.len(),
});
}
if items.iter().all(|item| item.is_array()) {
let max_cols = items
.iter()
.filter_map(|item| item.as_array().map(|row| row.len()))
.max()
.unwrap_or(0);
if max_cols == 0 {
return Err("Import file has no columns".to_string());
}
let columns = (0..max_cols)
.map(|index| format!("column_{}", index + 1))
.collect::<Vec<_>>();
let rows = items
.iter()
.take(preview_limit)
.map(|item| {
let arr = item.as_array().expect("checked array JSON row");
(0..max_cols)
.map(|index| arr.get(index).cloned().unwrap_or(serde_json::Value::Null))
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
return Ok(ParsedImportFile {
columns,
rows,
total_rows: items.len(),
});
}
Err("JSON rows must all be objects or all be arrays".to_string())
}
fn xlsx_cell_value(cell: &Data) -> serde_json::Value {
match cell {
Data::Empty => serde_json::Value::Null,
Data::String(s) => csv_value(s),
Data::Float(n) => serde_json::Number::from_f64(*n)
.map(serde_json::Value::Number)
.unwrap_or(serde_json::Value::Null),
Data::Int(n) => serde_json::Value::Number((*n).into()),
Data::Bool(v) => serde_json::Value::Bool(*v),
Data::DateTime(v) => serde_json::Value::String(v.to_string()),
Data::DateTimeIso(v) => serde_json::Value::String(v.clone()),
Data::DurationIso(v) => serde_json::Value::String(v.clone()),
Data::Error(v) => serde_json::Value::String(v.to_string()),
}
}
fn xlsx_cell_label(cell: &Data) -> String {
match cell {
Data::Empty => String::new(),
Data::String(s) => s.clone(),
Data::Float(n) => n.to_string(),
Data::Int(n) => n.to_string(),
Data::Bool(v) => v.to_string(),
Data::DateTime(v) => v.to_string(),
Data::DateTimeIso(v) => v.clone(),
Data::DurationIso(v) => v.clone(),
Data::Error(v) => v.to_string(),
}
}
fn parse_xlsx_file(path: &str, preview_limit: usize) -> Result<ParsedImportFile, String> {
let mut workbook = open_workbook_auto(path).map_err(|e| e.to_string())?;
let sheet_name = workbook
.sheet_names()
.first()
.cloned()
.ok_or_else(|| "Workbook has no sheets".to_string())?;
let range = workbook
.worksheet_range(&sheet_name)
.map_err(|e| e.to_string())?;
let mut rows_iter = range.rows();
let header = rows_iter
.next()
.ok_or_else(|| "Import file has no rows".to_string())?;
let columns = header
.iter()
.enumerate()
.map(|(index, cell)| normalize_header(&xlsx_cell_label(cell), index))
.collect::<Vec<_>>();
if columns.is_empty() {
return Err("Import file has no columns".to_string());
}
let mut rows = Vec::new();
let mut total_rows = 0;
for source_row in rows_iter {
total_rows += 1;
if rows.len() >= preview_limit {
continue;
}
let mut row = Vec::with_capacity(columns.len());
for index in 0..columns.len() {
row.push(
source_row
.get(index)
.map(xlsx_cell_value)
.unwrap_or(serde_json::Value::Null),
);
}
rows.push(row);
}
Ok(ParsedImportFile {
columns,
rows,
total_rows,
})
}
fn parse_import_file(path: &str, preview_limit: usize) -> Result<ParsedImportFile, String> {
match import_file_kind(path)? {
ImportFileKind::Csv => {
let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
parse_csv_bytes(&bytes, preview_limit)
}
ImportFileKind::Tsv => {
let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
parse_delimited_bytes(&bytes, b'\t', preview_limit)
}
ImportFileKind::Json => {
let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
parse_json_bytes(&bytes, preview_limit)
}
ImportFileKind::Xlsx => parse_xlsx_file(path, preview_limit),
}
}
fn mapping_indexes(
data: &ParsedImportFile,
mappings: &[TableImportColumnMapping],
) -> Result<Vec<(usize, String)>, String> {
if mappings.is_empty() {
return Err("No columns mapped for import".to_string());
}
let mut mapped = Vec::new();
let mut target_seen = HashSet::new();
for mapping in mappings {
let source_index = data
.columns
.iter()
.position(|column| column == &mapping.source_column)
.ok_or_else(|| format!("Source column not found: {}", mapping.source_column))?;
if mapping.target_column.trim().is_empty() {
return Err("Target column cannot be empty".to_string());
}
if !target_seen.insert(mapping.target_column.clone()) {
return Err(format!(
"Target column mapped more than once: {}",
mapping.target_column
));
}
mapped.push((source_index, mapping.target_column.clone()));
}
Ok(mapped)
}
fn build_import_insert_batches(
data: &ParsedImportFile,
mappings: &[TableImportColumnMapping],
table: &str,
schema: &str,
db_type: &DatabaseType,
batch_size: usize,
) -> Result<Vec<ImportSqlBatch>, String> {
let mapped = mapping_indexes(data, mappings)?;
let columns = mapped
.iter()
.map(|(_, target)| target.clone())
.collect::<Vec<_>>();
let batch_size = batch_size.max(1);
let mut batches = Vec::new();
for chunk in data.rows.chunks(batch_size) {
let rows = chunk
.iter()
.map(|row| {
mapped
.iter()
.map(|(source_index, _)| {
row.get(*source_index)
.cloned()
.unwrap_or(serde_json::Value::Null)
})
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
let sql = generate_insert(&columns, &rows, table, schema, db_type);
if !sql.trim().is_empty() {
batches.push(ImportSqlBatch {
sql,
row_count: chunk.len(),
});
}
}
Ok(batches)
}
fn truncate_sql(table: &str, schema: &str, db_type: &DatabaseType) -> String {
let full_table = qualified_table(table, schema, db_type);
match db_type {
DatabaseType::Sqlite => format!("DELETE FROM {full_table}"),
_ => format!("TRUNCATE TABLE {full_table}"),
}
}
fn emit_progress(app: &AppHandle, progress: TableImportProgress) {
let _ = app.emit("table-import-progress", progress);
}
async fn is_cancelled(import_id: &str) -> bool {
CANCELLED_IMPORTS.read().await.contains(import_id)
}
async fn clear_cancelled(import_id: &str) {
CANCELLED_IMPORTS.write().await.remove(import_id);
}
#[tauri::command]
pub async fn preview_table_import_file(file_path: String) -> Result<TableImportPreview, String> {
let kind = import_file_kind(&file_path)?;
let parsed = parse_import_file(&file_path, DEFAULT_PREVIEW_LIMIT)?;
let metadata = std::fs::metadata(&file_path).map_err(|e| e.to_string())?;
let file_name = Path::new(&file_path)
.file_name()
.and_then(|name| name.to_str())
.unwrap_or(&file_path)
.to_string();
Ok(TableImportPreview {
file_name,
file_path,
file_type: kind.label().to_string(),
size_bytes: metadata.len(),
columns: parsed.columns,
rows: parsed.rows,
total_rows: parsed.total_rows,
})
}
#[tauri::command]
pub async fn import_table_file(
app: AppHandle,
state: State<'_, Arc<AppState>>,
request: TableImportRequest,
) -> Result<TableImportSummary, String> {
clear_cancelled(&request.import_id).await;
let db_type = get_db_type(&state, &request.connection_id).await?;
let pool_key = if request.database.is_empty() {
request.connection_id.clone()
} else {
state
.get_or_create_pool(&request.connection_id, Some(&request.database))
.await?
};
let batch_size = if request.batch_size == 0 {
DEFAULT_BATCH_SIZE
} else {
request.batch_size
};
let parsed = match parse_import_file(&request.file_path, usize::MAX) {
Ok(parsed) => parsed,
Err(error) => {
emit_progress(
&app,
TableImportProgress {
import_id: request.import_id.clone(),
status: TableImportStatus::Error,
rows_imported: 0,
total_rows: 0,
error: Some(error.clone()),
},
);
return Err(error);
}
};
let total_rows = parsed.total_rows;
emit_progress(
&app,
TableImportProgress {
import_id: request.import_id.clone(),
status: TableImportStatus::Running,
rows_imported: 0,
total_rows,
error: None,
},
);
let batches = match build_import_insert_batches(
&parsed,
&request.mappings,
&request.table,
&request.schema,
&db_type,
batch_size,
) {
Ok(batches) => batches,
Err(error) => {
emit_progress(
&app,
TableImportProgress {
import_id: request.import_id.clone(),
status: TableImportStatus::Error,
rows_imported: 0,
total_rows,
error: Some(error.clone()),
},
);
return Err(error);
}
};
if matches!(request.mode, TableImportMode::Truncate) {
let sql = truncate_sql(&request.table, &request.schema, &db_type);
if let Err(error) = execute_on_pool(&state, &pool_key, &sql).await {
emit_progress(
&app,
TableImportProgress {
import_id: request.import_id.clone(),
status: TableImportStatus::Error,
rows_imported: 0,
total_rows,
error: Some(error.clone()),
},
);
return Err(error);
}
}
let mut rows_imported = 0;
for batch in batches {
if is_cancelled(&request.import_id).await {
emit_progress(
&app,
TableImportProgress {
import_id: request.import_id.clone(),
status: TableImportStatus::Cancelled,
rows_imported,
total_rows,
error: None,
},
);
clear_cancelled(&request.import_id).await;
return Err("Import cancelled".to_string());
}
if let Err(error) = execute_on_pool(&state, &pool_key, &batch.sql).await {
emit_progress(
&app,
TableImportProgress {
import_id: request.import_id.clone(),
status: TableImportStatus::Error,
rows_imported,
total_rows,
error: Some(error.clone()),
},
);
return Err(error);
}
rows_imported = (rows_imported + batch.row_count).min(total_rows);
emit_progress(
&app,
TableImportProgress {
import_id: request.import_id.clone(),
status: TableImportStatus::Running,
rows_imported,
total_rows,
error: None,
},
);
}
emit_progress(
&app,
TableImportProgress {
import_id: request.import_id.clone(),
status: TableImportStatus::Done,
rows_imported,
total_rows,
error: None,
},
);
clear_cancelled(&request.import_id).await;
Ok(TableImportSummary {
import_id: request.import_id,
rows_imported,
total_rows,
})
}
#[tauri::command]
pub async fn cancel_table_import(import_id: String) -> Result<bool, String> {
CANCELLED_IMPORTS.write().await.insert(import_id);
Ok(true)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::models::connection::DatabaseType;
#[test]
fn parses_csv_headers_and_preview_rows() {
let parsed = parse_csv_bytes(b"id,name,active\n1,Ada,true\n2,,false\n", 10).unwrap();
assert_eq!(parsed.columns, vec!["id", "name", "active"]);
assert_eq!(parsed.total_rows, 2);
assert_eq!(
parsed.rows[0],
vec![
serde_json::Value::String("1".to_string()),
serde_json::Value::String("Ada".to_string()),
serde_json::Value::String("true".to_string()),
]
);
assert_eq!(
parsed.rows[1],
vec![
serde_json::Value::String("2".to_string()),
serde_json::Value::Null,
serde_json::Value::String("false".to_string()),
]
);
}
#[test]
fn parses_tsv_with_tab_delimiter() {
let parsed = parse_delimited_bytes(b"id\tname\n1\tAda\n", b'\t', 10).unwrap();
assert_eq!(parsed.columns, vec!["id", "name"]);
assert_eq!(parsed.total_rows, 1);
assert_eq!(
parsed.rows[0],
vec![
serde_json::Value::String("1".to_string()),
serde_json::Value::String("Ada".to_string()),
]
);
}
#[test]
fn parses_json_array_objects_with_union_columns() {
let parsed =
parse_json_bytes(br#"[{"id":1,"name":"Ada"},{"id":2,"active":true}]"#, 10).unwrap();
assert_eq!(parsed.columns, vec!["id", "name", "active"]);
assert_eq!(parsed.total_rows, 2);
assert_eq!(
parsed.rows[0],
vec![
serde_json::json!(1),
serde_json::json!("Ada"),
serde_json::Value::Null,
]
);
assert_eq!(
parsed.rows[1],
vec![
serde_json::json!(2),
serde_json::Value::Null,
serde_json::json!(true),
]
);
}
#[test]
fn builds_import_insert_batches_from_mapped_columns() {
let mappings = vec![
TableImportColumnMapping {
source_column: "id".to_string(),
target_column: "user_id".to_string(),
},
TableImportColumnMapping {
source_column: "name".to_string(),
target_column: "display_name".to_string(),
},
];
let data = ParsedImportFile {
columns: vec!["id".to_string(), "name".to_string(), "ignored".to_string()],
rows: vec![
vec![
serde_json::json!(1),
serde_json::json!("Ada"),
serde_json::json!("x"),
],
vec![
serde_json::json!(2),
serde_json::json!("O'Hara"),
serde_json::json!("y"),
],
vec![
serde_json::json!(3),
serde_json::Value::Null,
serde_json::json!("z"),
],
],
total_rows: 3,
};
let batches = build_import_insert_batches(
&data,
&mappings,
"users",
"public",
&DatabaseType::Postgres,
2,
)
.unwrap();
assert_eq!(batches, vec![
ImportSqlBatch {
sql: "INSERT INTO \"public\".\"users\" (\"user_id\", \"display_name\") VALUES\n(1, 'Ada'),\n(2, 'O''Hara')".to_string(),
row_count: 2,
},
ImportSqlBatch {
sql: "INSERT INTO \"public\".\"users\" (\"user_id\", \"display_name\") VALUES\n(3, NULL)".to_string(),
row_count: 1,
},
]);
}
}

View File

@ -50,7 +50,7 @@ pub enum TransferStatus {
Cancelled,
}
fn quote_identifier(name: &str, db_type: &DatabaseType) -> String {
pub(crate) fn quote_identifier(name: &str, db_type: &DatabaseType) -> String {
match db_type {
DatabaseType::Mysql | DatabaseType::ClickHouse | DatabaseType::Doris | DatabaseType::StarRocks => format!("`{}`", name.replace('`', "``")),
DatabaseType::SqlServer => format!("[{}]", name.replace(']', "]]")),
@ -58,7 +58,7 @@ fn quote_identifier(name: &str, db_type: &DatabaseType) -> String {
}
}
fn qualified_table(table: &str, schema: &str, db_type: &DatabaseType) -> String {
pub(crate) fn qualified_table(table: &str, schema: &str, db_type: &DatabaseType) -> String {
let qt = quote_identifier(table, db_type);
if schema.is_empty() {
qt
@ -251,7 +251,7 @@ fn generate_create_table_ddl(
ddl
}
fn generate_insert(
pub(crate) fn generate_insert(
columns: &[String],
rows: &[Vec<serde_json::Value>],
table: &str,
@ -312,7 +312,7 @@ fn count_sql(table: &str, schema: &str, db_type: &DatabaseType) -> String {
format!("SELECT COUNT(*) FROM {full_table}")
}
async fn execute_on_pool(
pub(crate) async fn execute_on_pool(
state: &AppState,
pool_key: &str,
sql: &str,
@ -399,7 +399,7 @@ async fn execute_on_pool(
}
}
async fn get_db_type(state: &AppState, connection_id: &str) -> Result<DatabaseType, String> {
pub(crate) async fn get_db_type(state: &AppState, connection_id: &str) -> Result<DatabaseType, String> {
let configs = state.configs.lock().await;
configs
.get(connection_id)

View File

@ -64,6 +64,9 @@ pub fn run() {
commands::sql_file::preview_sql_file,
commands::sql_file::execute_sql_file,
commands::sql_file::cancel_sql_file_execution,
commands::table_import::preview_table_import_file,
commands::table_import::import_table_file,
commands::table_import::cancel_table_import,
commands::redis_cmd::redis_list_databases,
commands::redis_cmd::redis_scan_keys,
commands::redis_cmd::redis_get_value,

View File

@ -1,7 +1,7 @@
<script setup lang="ts">
import { ref, computed, watch, onMounted, onUnmounted, nextTick, type Ref } from "vue";
import { useI18n } from "vue-i18n";
import { DatabaseZap, FilePlus2, Play, Loader2, Square, X, Globe, Moon, Sun, Upload, Download, Plus, History, Server, Table2, Database, Search, ShieldCheck, Bot, Pin, AlignLeft, CloudDownload, ArrowLeftRight, FileCode, Settings } from "lucide-vue-next";
import { DatabaseZap, FilePlus2, Play, Loader2, Square, X, Globe, Moon, Sun, Upload, Download, Plus, History, Server, Table2, Database, Search, ShieldCheck, Bot, Pin, AlignLeft, CloudDownload, ArrowLeftRight, FileCode, Settings, Sparkles } from "lucide-vue-next";
import { Splitpanes, Pane } from "splitpanes";
import "splitpanes/dist/splitpanes.css";
import { Button } from "@/components/ui/button";
@ -38,6 +38,7 @@ import DataTransferDialog from "@/components/transfer/DataTransferDialog.vue";
import SchemaDiffDialog from "@/components/diff/SchemaDiffDialog.vue";
import SqlFileExecutionDialog from "@/components/sql-file/SqlFileExecutionDialog.vue";
import SchemaDiagramDialog from "@/components/diagram/SchemaDiagramDialog.vue";
import TableImportDialog from "@/components/import/TableImportDialog.vue";
import type { ConnectionConfig } from "@/types/database";
import { useConnectionStore } from "@/stores/connectionStore";
import { useQueryStore } from "@/stores/queryStore";
@ -122,6 +123,7 @@ const showTransferDialog = ref(false);
const showSchemaDiffDialog = ref(false);
const showSqlFileDialog = ref(false);
const showDiagramDialog = ref(false);
const showTableImportDialog = ref(false);
const transferPrefillConnectionId = ref("");
const transferPrefillDatabase = ref("");
const schemaDiffPrefillConnectionId = ref("");
@ -132,6 +134,10 @@ const diagramPrefillConnectionId = ref("");
const diagramPrefillDatabase = ref("");
const diagramPrefillSchema = ref("");
const diagramFocusTableName = ref("");
const tableImportPrefillConnectionId = ref("");
const tableImportPrefillDatabase = ref("");
const tableImportPrefillSchema = ref("");
const tableImportPrefillTable = ref("");
const databaseOptions = ref<Record<string, string[]>>({});
const loadingDatabaseOptions = ref<Record<string, boolean>>({});
const checkingUpdates = ref(false);
@ -197,6 +203,17 @@ watch(() => connectionStore.diagramSource, (v) => {
}
});
watch(() => connectionStore.tableImportSource, (v) => {
if (v) {
tableImportPrefillConnectionId.value = v.connectionId;
tableImportPrefillDatabase.value = v.database;
tableImportPrefillSchema.value = v.schema ?? "";
tableImportPrefillTable.value = v.tableName;
showTableImportDialog.value = true;
connectionStore.tableImportSource = null;
}
});
function onConnectionConnectStarted(name: string) {
toast(t("connection.connecting", { name }), 30000);
}
@ -1257,6 +1274,13 @@ async function setupFileDrop() {
:prefill-schema="diagramPrefillSchema"
:focus-table-name="diagramFocusTableName"
/>
<TableImportDialog
v-model:open="showTableImportDialog"
:prefill-connection-id="tableImportPrefillConnectionId"
:prefill-database="tableImportPrefillDatabase"
:prefill-schema="tableImportPrefillSchema"
:prefill-table="tableImportPrefillTable"
/>
<Dialog v-model:open="showUpdateDialog">
<DialogContent class="sm:max-w-[520px]">
<DialogHeader>

View File

@ -0,0 +1,367 @@
<script setup lang="ts">
import { computed, ref, watch } from "vue";
import { useI18n } from "vue-i18n";
import { open as openFileDialog } from "@tauri-apps/plugin-dialog";
import {
Dialog, DialogHeader, DialogTitle, DialogFooter, DialogScrollContent,
} from "@/components/ui/dialog";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import {
Select, SelectContent, SelectItem, SelectTrigger, SelectValue,
} from "@/components/ui/select";
import {
Check, FileUp, Loader2, Square, Upload, X,
} from "lucide-vue-next";
import { useConnectionStore } from "@/stores/connectionStore";
import { useToast } from "@/composables/useToast";
import { autoMapImportColumns } from "@/lib/tableImport";
import type { ColumnInfo } from "@/types/database";
import * as api from "@/lib/tauri";
const { t } = useI18n();
const store = useConnectionStore();
const { toast } = useToast();
const open = defineModel<boolean>("open", { default: false });
const props = defineProps<{
prefillConnectionId?: string;
prefillDatabase?: string;
prefillSchema?: string;
prefillTable?: string;
}>();
const SKIP_VALUE = "__skip__";
const targetColumns = ref<ColumnInfo[]>([]);
const preview = ref<api.TableImportPreview | null>(null);
const columnMapping = ref<Record<string, string>>({});
const loadingTarget = ref(false);
const loadingPreview = ref(false);
const importMode = ref<api.TableImportMode>("append");
const batchSize = ref(500);
const running = ref(false);
const cancelling = ref(false);
const importId = ref("");
const progress = ref<api.TableImportProgress | null>(null);
const errorMessage = ref("");
const selectedConnection = computed(() =>
props.prefillConnectionId ? store.getConfig(props.prefillConnectionId) : undefined
);
const targetColumnNames = computed(() => targetColumns.value.map((column) => column.name));
const mappedColumns = computed<api.TableImportColumnMapping[]>(() => {
const currentPreview = preview.value;
if (!currentPreview) return [];
return currentPreview.columns
.map((sourceColumn) => ({
sourceColumn,
targetColumn: columnMapping.value[sourceColumn] ?? "",
}))
.filter((mapping) => mapping.targetColumn);
});
const mappedCount = computed(() => mappedColumns.value.length);
const canImport = computed(() =>
!!preview.value &&
!!props.prefillConnectionId &&
!!props.prefillTable &&
mappedColumns.value.length > 0 &&
!running.value
);
const progressPercent = computed(() => {
const p = progress.value;
if (!p || p.totalRows <= 0) return 0;
return Math.min(100, Math.round((p.rowsImported / p.totalRows) * 100));
});
const targetLabel = computed(() => {
const pieces = [selectedConnection.value?.name, props.prefillDatabase, props.prefillSchema, props.prefillTable]
.filter(Boolean);
return pieces.join(" / ");
});
function resetState() {
targetColumns.value = [];
preview.value = null;
columnMapping.value = {};
importMode.value = "append";
batchSize.value = 500;
running.value = false;
cancelling.value = false;
importId.value = "";
progress.value = null;
errorMessage.value = "";
}
function applyAutoMapping() {
const currentPreview = preview.value;
if (!currentPreview) return;
columnMapping.value = autoMapImportColumns(currentPreview.columns, targetColumnNames.value);
}
async function loadTargetColumns() {
if (!props.prefillConnectionId || !props.prefillDatabase || !props.prefillTable) return;
loadingTarget.value = true;
errorMessage.value = "";
try {
await store.ensureConnected(props.prefillConnectionId);
targetColumns.value = await api.getColumns(
props.prefillConnectionId,
props.prefillDatabase,
props.prefillSchema || props.prefillDatabase,
props.prefillTable,
);
applyAutoMapping();
} catch (e: any) {
errorMessage.value = String(e?.message || e);
} finally {
loadingTarget.value = false;
}
}
async function selectFile() {
const selected = await openFileDialog({
multiple: false,
filters: [
{ name: "Data files", extensions: ["csv", "tsv", "json", "xlsx", "xlsm", "xls"] },
{ name: "CSV", extensions: ["csv", "tsv"] },
{ name: "JSON", extensions: ["json"] },
{ name: "Excel", extensions: ["xlsx", "xlsm", "xls"] },
],
});
if (!selected || Array.isArray(selected)) return;
loadingPreview.value = true;
errorMessage.value = "";
try {
preview.value = await api.previewTableImportFile(selected);
applyAutoMapping();
} catch (e: any) {
preview.value = null;
columnMapping.value = {};
errorMessage.value = String(e?.message || e);
} finally {
loadingPreview.value = false;
}
}
function updateMapping(sourceColumn: string, value: any) {
const target = String(value);
columnMapping.value = {
...columnMapping.value,
[sourceColumn]: target === SKIP_VALUE ? "" : target,
};
}
function formatCell(value: unknown) {
if (value === null) return "NULL";
if (typeof value === "object") return JSON.stringify(value);
return String(value);
}
async function startImport() {
const currentPreview = preview.value;
if (!canImport.value || !currentPreview || !props.prefillConnectionId || !props.prefillTable) return;
running.value = true;
cancelling.value = false;
errorMessage.value = "";
importId.value = crypto.randomUUID();
progress.value = {
importId: importId.value,
status: "running",
rowsImported: 0,
totalRows: currentPreview.totalRows,
};
try {
const summary = await api.importTableFile({
importId: importId.value,
connectionId: props.prefillConnectionId,
database: props.prefillDatabase || "",
schema: props.prefillSchema || "",
table: props.prefillTable,
filePath: currentPreview.filePath,
mappings: mappedColumns.value,
mode: importMode.value,
batchSize: Math.max(1, Number(batchSize.value) || 500),
}, (nextProgress) => {
progress.value = nextProgress;
});
toast(t("tableImport.success", { count: summary.rowsImported }), 2500);
open.value = false;
} catch (e: any) {
errorMessage.value = String(e?.message || e);
} finally {
running.value = false;
cancelling.value = false;
}
}
async function cancelImport() {
if (!importId.value) return;
cancelling.value = true;
await api.cancelTableImport(importId.value);
}
watch(open, (value) => {
if (value) {
resetState();
void loadTargetColumns();
}
});
</script>
<template>
<Dialog v-model:open="open">
<DialogScrollContent class="sm:max-w-[760px]" :trap-focus="false" @interact-outside.prevent>
<DialogHeader>
<DialogTitle class="flex items-center gap-2">
<FileUp class="h-4 w-4" />
{{ t('tableImport.title') }}
</DialogTitle>
</DialogHeader>
<div class="space-y-4 py-2">
<div class="grid grid-cols-[1fr_auto] gap-2">
<div class="min-w-0 rounded-md border bg-muted/20 px-3 py-2">
<div class="truncate text-xs text-muted-foreground">{{ t('tableImport.target') }}</div>
<div class="truncate text-sm font-medium">{{ targetLabel || t('editor.noDatabase') }}</div>
</div>
<Button variant="outline" size="sm" :disabled="running || loadingPreview" @click="selectFile">
<Loader2 v-if="loadingPreview" class="mr-1.5 h-3.5 w-3.5 animate-spin" />
<Upload v-else class="mr-1.5 h-3.5 w-3.5" />
{{ t('tableImport.selectFile') }}
</Button>
</div>
<div v-if="preview" class="grid grid-cols-3 gap-2 text-xs">
<div class="rounded-md border px-3 py-2">
<div class="text-muted-foreground">{{ t('tableImport.file') }}</div>
<div class="truncate font-medium">{{ preview.fileName }}</div>
</div>
<div class="rounded-md border px-3 py-2">
<div class="text-muted-foreground">{{ t('tableImport.rows') }}</div>
<div class="font-medium">{{ preview.totalRows.toLocaleString() }}</div>
</div>
<div class="rounded-md border px-3 py-2">
<div class="text-muted-foreground">{{ t('tableImport.mapped') }}</div>
<div class="font-medium">{{ mappedCount }} / {{ preview.columns.length }}</div>
</div>
</div>
<div v-if="preview" class="grid grid-cols-[minmax(220px,280px)_1fr] gap-3">
<div class="rounded-md border">
<div class="border-b px-3 py-2 text-xs font-medium">{{ t('tableImport.mapping') }}</div>
<div class="max-h-[280px] overflow-auto p-2">
<div
v-for="sourceColumn in preview.columns"
:key="sourceColumn"
class="grid grid-cols-[1fr_1fr] items-center gap-2 py-1"
>
<div class="truncate font-mono text-xs" :title="sourceColumn">{{ sourceColumn }}</div>
<Select
:model-value="columnMapping[sourceColumn] || SKIP_VALUE"
@update:model-value="(value: any) => updateMapping(sourceColumn, value)"
>
<SelectTrigger class="h-7 text-xs">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem :value="SKIP_VALUE">{{ t('tableImport.skipColumn') }}</SelectItem>
<SelectItem v-for="column in targetColumns" :key="column.name" :value="column.name">
{{ column.name }}
</SelectItem>
</SelectContent>
</Select>
</div>
</div>
</div>
<div class="min-w-0 rounded-md border">
<div class="border-b px-3 py-2 text-xs font-medium">{{ t('tableImport.preview') }}</div>
<div class="max-h-[280px] overflow-auto">
<table class="min-w-full border-separate border-spacing-0 text-xs">
<thead class="sticky top-0 bg-background">
<tr>
<th
v-for="column in preview.columns"
:key="column"
class="border-b border-r px-2 py-1.5 text-left font-medium"
>
<span class="block max-w-[140px] truncate">{{ column }}</span>
</th>
</tr>
</thead>
<tbody>
<tr v-for="(row, rowIndex) in preview.rows" :key="rowIndex">
<td
v-for="(cell, colIndex) in row"
:key="colIndex"
class="max-w-[180px] border-b border-r px-2 py-1.5 font-mono"
:class="{ 'text-muted-foreground': cell === null }"
>
<span class="block truncate">{{ formatCell(cell) }}</span>
</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div v-if="preview" class="grid grid-cols-3 gap-3">
<div class="space-y-1.5">
<Label class="text-xs">{{ t('tableImport.mode') }}</Label>
<Select :model-value="importMode" @update:model-value="(value: any) => importMode = value">
<SelectTrigger class="h-8 text-xs">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="append">{{ t('tableImport.append') }}</SelectItem>
<SelectItem value="truncate">{{ t('tableImport.truncate') }}</SelectItem>
</SelectContent>
</Select>
</div>
<div class="space-y-1.5">
<Label class="text-xs">{{ t('transfer.batchSize') }}</Label>
<Input v-model.number="batchSize" type="number" min="1" class="h-8 text-xs" />
</div>
<div v-if="running || progress" class="space-y-1.5">
<Label class="text-xs">{{ t('tableImport.progress') }}</Label>
<div class="h-8 rounded-md border px-2 text-xs flex items-center gap-2">
<Loader2 v-if="running && !cancelling" class="h-3.5 w-3.5 animate-spin text-primary" />
<Square v-else-if="cancelling" class="h-3.5 w-3.5 fill-current text-destructive" />
<Check v-else class="h-3.5 w-3.5 text-emerald-600" />
<span class="truncate">
{{ progress?.rowsImported ?? 0 }} / {{ progress?.totalRows ?? preview.totalRows }} · {{ progressPercent }}%
</span>
</div>
</div>
</div>
<div v-if="loadingTarget" class="flex items-center gap-2 text-xs text-muted-foreground">
<Loader2 class="h-3.5 w-3.5 animate-spin" />
{{ t('common.loading') }}
</div>
<div v-if="errorMessage" class="rounded-md border border-destructive/30 bg-destructive/10 px-3 py-2 text-xs text-destructive">
{{ errorMessage }}
</div>
</div>
<DialogFooter>
<Button variant="outline" :disabled="running" @click="open = false">
<X class="mr-1.5 h-3.5 w-3.5" />
{{ t('dangerDialog.cancel') }}
</Button>
<Button v-if="running" variant="destructive" :disabled="cancelling" @click="cancelImport">
<Loader2 v-if="cancelling" class="mr-1.5 h-3.5 w-3.5 animate-spin" />
<Square v-else class="mr-1.5 h-3.5 w-3.5 fill-current" />
{{ t('sqlFile.cancel') }}
</Button>
<Button v-else :disabled="!canImport" @click="startImport">
<Upload class="mr-1.5 h-3.5 w-3.5" />
{{ t('tableImport.start') }}
</Button>
</DialogFooter>
</DialogScrollContent>
</Dialog>
</template>

View File

@ -5,7 +5,7 @@ import {
Database, Table, Columns3, Eye, ChevronRight, ChevronDown,
Loader2, FolderOpen, Trash2, TerminalSquare, RefreshCw,
Copy, TableProperties, Key, Link, Zap, ListTree, Pencil, Plug, Unplug,
Pin, ArrowRightLeft, Download, FileCode, Network,
Pin, ArrowRightLeft, Download, FileCode, Network, FileUp,
} from "lucide-vue-next";
import {
ContextMenu, ContextMenuContent, ContextMenuItem,
@ -43,6 +43,7 @@ const props = defineProps<{
const sqlFileUnsupportedTypes = new Set(["redis", "mongodb", "elasticsearch"]);
const diagramSupportedTypes = new Set(["mysql", "postgres", "sqlite", "sqlserver", "oracle", "redshift"]);
const tableImportSupportedTypes = new Set(["mysql", "postgres", "sqlite", "duckdb", "clickhouse", "sqlserver", "oracle", "doris", "starrocks", "redshift"]);
const isExportingDatabase = ref(false);
function currentDatabaseType(): DatabaseType | undefined {
@ -513,6 +514,17 @@ function openDiagram() {
};
}
function openTableImport() {
const node = props.node;
if (node.type !== "table" || !node.connectionId || !node.database) return;
connectionStore.tableImportSource = {
connectionId: node.connectionId,
database: node.database,
schema: node.schema,
tableName: node.label,
};
}
const canExpand = !leafTypes.has(props.node.type);
const canPin = computed(() => pinnableTypes.has(props.node.type));
const canOpenSqlFileExecution = computed(() => {
@ -523,6 +535,10 @@ const canOpenDiagram = computed(() => {
const config = props.node.connectionId ? connectionStore.getConfig(props.node.connectionId) : undefined;
return !!props.node.database && !!config && diagramSupportedTypes.has(config.db_type);
});
const canOpenTableImport = computed(() => {
const config = props.node.connectionId ? connectionStore.getConfig(props.node.connectionId) : undefined;
return props.node.type === "table" && !!props.node.database && !!config && tableImportSupportedTypes.has(config.db_type);
});
const isPinned = computed(() => props.node.pinned || connectionStore.isTreeNodePinned(props.node.id));
const hasTypeMenu = computed(() => {
const t = props.node.type;
@ -685,6 +701,9 @@ async function showMore() {
<ContextMenuItem v-if="canOpenDiagram" @click="openDiagram">
<Network class="w-4 h-4" /> {{ t('diagram.open') }}
</ContextMenuItem>
<ContextMenuItem v-if="canOpenTableImport" @click="openTableImport">
<FileUp class="w-4 h-4" /> {{ t('contextMenu.importData') }}
</ContextMenuItem>
<ContextMenuSeparator />
<ContextMenuSub>
<ContextMenuSubTrigger>

View File

@ -226,6 +226,7 @@ export default {
exportDatabase: "Export Database",
exportDatabaseSuccess: "Exported {count} tables, up to {limit} rows each",
exportDatabaseFailed: "Failed to export database: {message}",
importData: "Import Data",
exportData: "Export Data",
exportStructure: "Export Structure",
},
@ -349,6 +350,23 @@ export default {
overallProgress: "Overall progress",
dataTransfer: "Data Transfer",
},
tableImport: {
title: "Import Table Data",
target: "Target table",
selectFile: "Select File",
file: "File",
rows: "Rows",
mapped: "Mapped",
mapping: "Column Mapping",
skipColumn: "Skip",
preview: "Preview",
mode: "Import mode",
append: "Append data",
truncate: "Truncate then import",
progress: "Progress",
start: "Start Import",
success: "Imported {count} rows",
},
sqlFile: {
title: "Execute SQL File",
file: "File",

View File

@ -226,6 +226,7 @@ export default {
exportDatabase: "导出数据库",
exportDatabaseSuccess: "已导出 {count} 张表,每表最多 {limit} 行",
exportDatabaseFailed: "导出数据库失败:{message}",
importData: "导入数据",
exportData: "导出数据",
exportStructure: "导出表结构",
},
@ -349,6 +350,23 @@ export default {
overallProgress: "整体进度",
dataTransfer: "数据传输",
},
tableImport: {
title: "导入表数据",
target: "目标表",
selectFile: "选择文件",
file: "文件",
rows: "行数",
mapped: "已映射",
mapping: "字段映射",
skipColumn: "跳过",
preview: "预览",
mode: "导入模式",
append: "追加数据",
truncate: "清空后导入",
progress: "进度",
start: "开始导入",
success: "已导入 {count} 行",
},
sqlFile: {
title: "执行 SQL 文件",
file: "文件",

22
src/lib/tableImport.ts Normal file
View File

@ -0,0 +1,22 @@
export const IMPORT_SKIP_TARGET = "";
export function normalizeImportColumnName(name: string): string {
return name.trim().toLowerCase().replace(/[_-]+/g, " ").replace(/\s+/g, " ");
}
export function autoMapImportColumns(
sourceColumns: string[],
targetColumns: string[],
): Record<string, string> {
const exactTargets = new Map(targetColumns.map((column) => [column, column]));
const normalizedTargets = new Map(
targetColumns.map((column) => [normalizeImportColumnName(column), column]),
);
return Object.fromEntries(
sourceColumns.map((source) => [
source,
exactTargets.get(source) ?? normalizedTargets.get(normalizeImportColumnName(source)) ?? IMPORT_SKIP_TARGET,
]),
);
}

View File

@ -409,3 +409,76 @@ export async function startTransfer(
export async function cancelTransfer(transferId: string): Promise<void> {
return invoke("cancel_transfer", { transferId });
}
// --- Table File Import ---
export type TableImportMode = "append" | "truncate";
export type TableImportStatus = "running" | "done" | "error" | "cancelled";
export interface TableImportColumnMapping {
sourceColumn: string;
targetColumn: string;
}
export interface TableImportPreview {
fileName: string;
filePath: string;
fileType: string;
sizeBytes: number;
columns: string[];
rows: unknown[][];
totalRows: number;
}
export interface TableImportRequest {
importId: string;
connectionId: string;
database: string;
schema: string;
table: string;
filePath: string;
mappings: TableImportColumnMapping[];
mode: TableImportMode;
batchSize: number;
}
export interface TableImportSummary {
importId: string;
rowsImported: number;
totalRows: number;
}
export interface TableImportProgress {
importId: string;
status: TableImportStatus;
rowsImported: number;
totalRows: number;
error?: string | null;
}
export async function previewTableImportFile(filePath: string): Promise<TableImportPreview> {
return invoke("preview_table_import_file", { filePath });
}
export async function importTableFile(
request: TableImportRequest,
onProgress: (progress: TableImportProgress) => void,
): Promise<TableImportSummary> {
const unlisten: UnlistenFn = await listen<TableImportProgress>("table-import-progress", (event) => {
if (event.payload.importId === request.importId) {
onProgress(event.payload);
if (event.payload.status === "done" || event.payload.status === "error" || event.payload.status === "cancelled") {
unlisten();
}
}
});
try {
return await invoke("import_table_file", { request });
} catch (e) {
unlisten();
throw e;
}
}
export async function cancelTableImport(importId: string): Promise<boolean> {
return invoke("cancel_table_import", { importId });
}

View File

@ -20,6 +20,7 @@ export const useConnectionStore = defineStore("connection", () => {
const schemaDiffSource = ref<{ connectionId: string; database: string } | null>(null);
const sqlFileSource = ref<{ connectionId: string; database: string } | null>(null);
const diagramSource = ref<{ connectionId: string; database: string; schema?: string; tableName?: string } | null>(null);
const tableImportSource = ref<{ connectionId: string; database: string; schema?: string; tableName: string } | null>(null);
function startEditing(id: string) {
editingConnectionId.value = id;
@ -657,5 +658,6 @@ export const useConnectionStore = defineStore("connection", () => {
schemaDiffSource,
sqlFileSource,
diagramSource,
tableImportSource,
};
});

26
tests/tableImport.test.ts Normal file
View File

@ -0,0 +1,26 @@
import { strict as assert } from "node:assert";
import test from "node:test";
import {
autoMapImportColumns,
normalizeImportColumnName,
} from "../src/lib/tableImport.ts";
test("normalizes import column names for matching", () => {
assert.equal(normalizeImportColumnName(" User ID "), "user id");
assert.equal(normalizeImportColumnName("user_id"), "user id");
assert.equal(normalizeImportColumnName("USER-ID"), "user id");
});
test("auto maps source columns to matching target columns and skips unknown columns", () => {
const mapping = autoMapImportColumns(
["id", "Name", "created-at", "ignored"],
["user_id", "name", "created_at"],
);
assert.deepEqual(mapping, {
id: "",
Name: "name",
"created-at": "created_at",
ignored: "",
});
});