diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 70f157b06..f198bca8b 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -529,6 +529,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi_simd" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a49e05797ca52e312a0c658938b7d00693ef037799ef7187678f212d7684cf" +dependencies = [ + "debug_unsafe", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -872,6 +881,23 @@ dependencies = [ "system-deps", ] +[[package]] +name = "calamine" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a9acfeb1555aa8def91fe8ff208aadaea850c109968ec35ac965edbe7d210b" +dependencies = [ + "atoi_simd", + "byteorder", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml 0.37.5", + "serde", + "zip 4.6.1", +] + [[package]] name = "camino" version = "1.2.2" @@ -1061,6 +1087,15 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + [[package]] name = "combine" version = "4.6.7" @@ -1412,6 +1447,27 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctor" version = "0.2.9" @@ -1551,7 +1607,9 @@ name = "dbx" version = "0.3.6" dependencies = [ "anyhow", + "calamine", "chrono", + "csv", "duckdb", "futures", "keyring", @@ -1580,6 +1638,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + [[package]] name = "delegate" version = "0.13.5" @@ -2092,6 +2156,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fast-float2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + [[package]] name = "fastrand" version = "2.4.1" @@ -3669,7 +3739,7 @@ dependencies = [ "serde_json", "tar", "vcpkg", - "zip", + "zip 6.0.0", ] [[package]] @@ -5004,7 +5074,7 @@ checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" dependencies = [ "base64 0.22.1", "indexmap 2.14.0", - "quick-xml", + "quick-xml 0.39.2", "serde", "time", ] @@ -5242,6 +5312,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quick-xml" version = "0.39.2" @@ -9581,6 +9661,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.14.0", + "memchr", + "zopfli", +] + [[package]] name = "zip" version = "6.0.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 1bfc2f064..1fe17155a 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -49,3 +49,5 @@ oracle-rs = "0.1" percent-encoding = "2" russh = "0.60" keyring = { version = "3.6.3", features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] } +csv = "1.4.0" +calamine = "0.30.1" diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 718c1c19c..d0ff39c18 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -8,5 +8,6 @@ pub mod query_cancel; pub mod redis_cmd; pub mod schema; pub mod sql_file; +pub mod table_import; pub mod transfer; pub mod update; diff --git a/src-tauri/src/commands/table_import.rs b/src-tauri/src/commands/table_import.rs new file mode 100644 index 000000000..672d77dfd --- /dev/null +++ b/src-tauri/src/commands/table_import.rs @@ -0,0 +1,770 @@ +use std::collections::HashSet; +use std::path::Path; +use std::sync::Arc; + +use calamine::{open_workbook_auto, Data, Reader}; +use serde::{Deserialize, Serialize}; +use tauri::{AppHandle, Emitter, State}; +use tokio::sync::RwLock; + +use crate::commands::connection::AppState; +use crate::commands::transfer::{execute_on_pool, generate_insert, get_db_type, qualified_table}; +use crate::models::connection::DatabaseType; + +const DEFAULT_PREVIEW_LIMIT: usize = 50; +const DEFAULT_BATCH_SIZE: usize = 500; + +static CANCELLED_IMPORTS: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| RwLock::new(HashSet::new())); + +#[derive(Debug, Clone)] +struct ParsedImportFile { + columns: Vec, + rows: Vec>, + total_rows: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ImportSqlBatch { + sql: String, + row_count: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TableImportColumnMapping { + pub source_column: String, + pub target_column: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TableImportMode { + Append, + Truncate, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TableImportRequest { + pub import_id: String, + pub connection_id: String, + pub database: String, + pub schema: String, + pub table: String, + pub file_path: String, + pub mappings: Vec, + pub mode: TableImportMode, + pub batch_size: usize, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TableImportPreview { + pub file_name: String, + pub file_path: String, + pub file_type: String, + pub size_bytes: u64, + pub columns: Vec, + pub rows: Vec>, + pub total_rows: usize, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TableImportSummary { + pub import_id: String, + pub rows_imported: usize, + pub total_rows: usize, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TableImportProgress { + pub import_id: String, + pub status: TableImportStatus, + pub rows_imported: usize, + pub total_rows: usize, + pub error: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum TableImportStatus { + Running, + Done, + Error, + Cancelled, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ImportFileKind { + Csv, + Tsv, + Json, + Xlsx, +} + +impl ImportFileKind { + fn label(self) -> &'static str { + match self { + ImportFileKind::Csv => "csv", + ImportFileKind::Tsv => "tsv", + ImportFileKind::Json => "json", + ImportFileKind::Xlsx => "xlsx", + } + } +} + +fn import_file_kind(path: &str) -> Result { + let lower = path.to_lowercase(); + if lower.ends_with(".csv") { + Ok(ImportFileKind::Csv) + } else if lower.ends_with(".tsv") { + Ok(ImportFileKind::Tsv) + } else if lower.ends_with(".json") { + Ok(ImportFileKind::Json) + } else if lower.ends_with(".xlsx") || lower.ends_with(".xlsm") || lower.ends_with(".xls") { + Ok(ImportFileKind::Xlsx) + } else { + Err("Unsupported import file type".to_string()) + } +} + +fn normalize_header(value: &str, index: usize) -> String { + let trimmed = value.trim(); + if trimmed.is_empty() { + format!("column_{}", index + 1) + } else { + trimmed.to_string() + } +} + +fn csv_value(value: &str) -> serde_json::Value { + if value.is_empty() { + serde_json::Value::Null + } else { + serde_json::Value::String(value.to_string()) + } +} + +fn parse_delimited_bytes( + bytes: &[u8], + delimiter: u8, + preview_limit: usize, +) -> Result { + let mut reader = csv::ReaderBuilder::new() + .delimiter(delimiter) + .flexible(true) + .from_reader(bytes); + let columns = reader + .headers() + .map_err(|e| e.to_string())? + .iter() + .enumerate() + .map(|(index, header)| normalize_header(header, index)) + .collect::>(); + if columns.is_empty() { + return Err("Import file has no columns".to_string()); + } + + let mut rows = Vec::new(); + let mut total_rows = 0; + for record in reader.records() { + let record = record.map_err(|e| e.to_string())?; + total_rows += 1; + if rows.len() >= preview_limit { + continue; + } + let mut row = Vec::with_capacity(columns.len()); + for index in 0..columns.len() { + row.push( + record + .get(index) + .map(csv_value) + .unwrap_or(serde_json::Value::Null), + ); + } + rows.push(row); + } + + Ok(ParsedImportFile { + columns, + rows, + total_rows, + }) +} + +fn parse_csv_bytes(bytes: &[u8], preview_limit: usize) -> Result { + parse_delimited_bytes(bytes, b',', preview_limit) +} + +fn parse_json_bytes(bytes: &[u8], preview_limit: usize) -> Result { + let value: serde_json::Value = serde_json::from_slice(bytes).map_err(|e| e.to_string())?; + let items = match value { + serde_json::Value::Array(items) => items, + serde_json::Value::Object(_) => vec![value], + _ => return Err("JSON import must be an object or an array".to_string()), + }; + if items.is_empty() { + return Err("Import file has no rows".to_string()); + } + + if items.iter().all(|item| item.is_object()) { + let mut columns = Vec::new(); + for item in &items { + if let Some(obj) = item.as_object() { + for key in obj.keys() { + if !columns.contains(key) { + columns.push(key.clone()); + } + } + } + } + if columns.is_empty() { + return Err("Import file has no columns".to_string()); + } + let rows = items + .iter() + .take(preview_limit) + .map(|item| { + let obj = item.as_object().expect("checked object JSON row"); + columns + .iter() + .map(|column| obj.get(column).cloned().unwrap_or(serde_json::Value::Null)) + .collect::>() + }) + .collect::>(); + return Ok(ParsedImportFile { + columns, + rows, + total_rows: items.len(), + }); + } + + if items.iter().all(|item| item.is_array()) { + let max_cols = items + .iter() + .filter_map(|item| item.as_array().map(|row| row.len())) + .max() + .unwrap_or(0); + if max_cols == 0 { + return Err("Import file has no columns".to_string()); + } + let columns = (0..max_cols) + .map(|index| format!("column_{}", index + 1)) + .collect::>(); + let rows = items + .iter() + .take(preview_limit) + .map(|item| { + let arr = item.as_array().expect("checked array JSON row"); + (0..max_cols) + .map(|index| arr.get(index).cloned().unwrap_or(serde_json::Value::Null)) + .collect::>() + }) + .collect::>(); + return Ok(ParsedImportFile { + columns, + rows, + total_rows: items.len(), + }); + } + + Err("JSON rows must all be objects or all be arrays".to_string()) +} + +fn xlsx_cell_value(cell: &Data) -> serde_json::Value { + match cell { + Data::Empty => serde_json::Value::Null, + Data::String(s) => csv_value(s), + Data::Float(n) => serde_json::Number::from_f64(*n) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null), + Data::Int(n) => serde_json::Value::Number((*n).into()), + Data::Bool(v) => serde_json::Value::Bool(*v), + Data::DateTime(v) => serde_json::Value::String(v.to_string()), + Data::DateTimeIso(v) => serde_json::Value::String(v.clone()), + Data::DurationIso(v) => serde_json::Value::String(v.clone()), + Data::Error(v) => serde_json::Value::String(v.to_string()), + } +} + +fn xlsx_cell_label(cell: &Data) -> String { + match cell { + Data::Empty => String::new(), + Data::String(s) => s.clone(), + Data::Float(n) => n.to_string(), + Data::Int(n) => n.to_string(), + Data::Bool(v) => v.to_string(), + Data::DateTime(v) => v.to_string(), + Data::DateTimeIso(v) => v.clone(), + Data::DurationIso(v) => v.clone(), + Data::Error(v) => v.to_string(), + } +} + +fn parse_xlsx_file(path: &str, preview_limit: usize) -> Result { + let mut workbook = open_workbook_auto(path).map_err(|e| e.to_string())?; + let sheet_name = workbook + .sheet_names() + .first() + .cloned() + .ok_or_else(|| "Workbook has no sheets".to_string())?; + let range = workbook + .worksheet_range(&sheet_name) + .map_err(|e| e.to_string())?; + let mut rows_iter = range.rows(); + let header = rows_iter + .next() + .ok_or_else(|| "Import file has no rows".to_string())?; + let columns = header + .iter() + .enumerate() + .map(|(index, cell)| normalize_header(&xlsx_cell_label(cell), index)) + .collect::>(); + if columns.is_empty() { + return Err("Import file has no columns".to_string()); + } + + let mut rows = Vec::new(); + let mut total_rows = 0; + for source_row in rows_iter { + total_rows += 1; + if rows.len() >= preview_limit { + continue; + } + let mut row = Vec::with_capacity(columns.len()); + for index in 0..columns.len() { + row.push( + source_row + .get(index) + .map(xlsx_cell_value) + .unwrap_or(serde_json::Value::Null), + ); + } + rows.push(row); + } + + Ok(ParsedImportFile { + columns, + rows, + total_rows, + }) +} + +fn parse_import_file(path: &str, preview_limit: usize) -> Result { + match import_file_kind(path)? { + ImportFileKind::Csv => { + let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + parse_csv_bytes(&bytes, preview_limit) + } + ImportFileKind::Tsv => { + let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + parse_delimited_bytes(&bytes, b'\t', preview_limit) + } + ImportFileKind::Json => { + let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + parse_json_bytes(&bytes, preview_limit) + } + ImportFileKind::Xlsx => parse_xlsx_file(path, preview_limit), + } +} + +fn mapping_indexes( + data: &ParsedImportFile, + mappings: &[TableImportColumnMapping], +) -> Result, String> { + if mappings.is_empty() { + return Err("No columns mapped for import".to_string()); + } + let mut mapped = Vec::new(); + let mut target_seen = HashSet::new(); + for mapping in mappings { + let source_index = data + .columns + .iter() + .position(|column| column == &mapping.source_column) + .ok_or_else(|| format!("Source column not found: {}", mapping.source_column))?; + if mapping.target_column.trim().is_empty() { + return Err("Target column cannot be empty".to_string()); + } + if !target_seen.insert(mapping.target_column.clone()) { + return Err(format!( + "Target column mapped more than once: {}", + mapping.target_column + )); + } + mapped.push((source_index, mapping.target_column.clone())); + } + Ok(mapped) +} + +fn build_import_insert_batches( + data: &ParsedImportFile, + mappings: &[TableImportColumnMapping], + table: &str, + schema: &str, + db_type: &DatabaseType, + batch_size: usize, +) -> Result, String> { + let mapped = mapping_indexes(data, mappings)?; + let columns = mapped + .iter() + .map(|(_, target)| target.clone()) + .collect::>(); + let batch_size = batch_size.max(1); + let mut batches = Vec::new(); + + for chunk in data.rows.chunks(batch_size) { + let rows = chunk + .iter() + .map(|row| { + mapped + .iter() + .map(|(source_index, _)| { + row.get(*source_index) + .cloned() + .unwrap_or(serde_json::Value::Null) + }) + .collect::>() + }) + .collect::>(); + let sql = generate_insert(&columns, &rows, table, schema, db_type); + if !sql.trim().is_empty() { + batches.push(ImportSqlBatch { + sql, + row_count: chunk.len(), + }); + } + } + + Ok(batches) +} + +fn truncate_sql(table: &str, schema: &str, db_type: &DatabaseType) -> String { + let full_table = qualified_table(table, schema, db_type); + match db_type { + DatabaseType::Sqlite => format!("DELETE FROM {full_table}"), + _ => format!("TRUNCATE TABLE {full_table}"), + } +} + +fn emit_progress(app: &AppHandle, progress: TableImportProgress) { + let _ = app.emit("table-import-progress", progress); +} + +async fn is_cancelled(import_id: &str) -> bool { + CANCELLED_IMPORTS.read().await.contains(import_id) +} + +async fn clear_cancelled(import_id: &str) { + CANCELLED_IMPORTS.write().await.remove(import_id); +} + +#[tauri::command] +pub async fn preview_table_import_file(file_path: String) -> Result { + let kind = import_file_kind(&file_path)?; + let parsed = parse_import_file(&file_path, DEFAULT_PREVIEW_LIMIT)?; + let metadata = std::fs::metadata(&file_path).map_err(|e| e.to_string())?; + let file_name = Path::new(&file_path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(&file_path) + .to_string(); + + Ok(TableImportPreview { + file_name, + file_path, + file_type: kind.label().to_string(), + size_bytes: metadata.len(), + columns: parsed.columns, + rows: parsed.rows, + total_rows: parsed.total_rows, + }) +} + +#[tauri::command] +pub async fn import_table_file( + app: AppHandle, + state: State<'_, Arc>, + request: TableImportRequest, +) -> Result { + clear_cancelled(&request.import_id).await; + let db_type = get_db_type(&state, &request.connection_id).await?; + let pool_key = if request.database.is_empty() { + request.connection_id.clone() + } else { + state + .get_or_create_pool(&request.connection_id, Some(&request.database)) + .await? + }; + let batch_size = if request.batch_size == 0 { + DEFAULT_BATCH_SIZE + } else { + request.batch_size + }; + + let parsed = match parse_import_file(&request.file_path, usize::MAX) { + Ok(parsed) => parsed, + Err(error) => { + emit_progress( + &app, + TableImportProgress { + import_id: request.import_id.clone(), + status: TableImportStatus::Error, + rows_imported: 0, + total_rows: 0, + error: Some(error.clone()), + }, + ); + return Err(error); + } + }; + + let total_rows = parsed.total_rows; + emit_progress( + &app, + TableImportProgress { + import_id: request.import_id.clone(), + status: TableImportStatus::Running, + rows_imported: 0, + total_rows, + error: None, + }, + ); + + let batches = match build_import_insert_batches( + &parsed, + &request.mappings, + &request.table, + &request.schema, + &db_type, + batch_size, + ) { + Ok(batches) => batches, + Err(error) => { + emit_progress( + &app, + TableImportProgress { + import_id: request.import_id.clone(), + status: TableImportStatus::Error, + rows_imported: 0, + total_rows, + error: Some(error.clone()), + }, + ); + return Err(error); + } + }; + + if matches!(request.mode, TableImportMode::Truncate) { + let sql = truncate_sql(&request.table, &request.schema, &db_type); + if let Err(error) = execute_on_pool(&state, &pool_key, &sql).await { + emit_progress( + &app, + TableImportProgress { + import_id: request.import_id.clone(), + status: TableImportStatus::Error, + rows_imported: 0, + total_rows, + error: Some(error.clone()), + }, + ); + return Err(error); + } + } + + let mut rows_imported = 0; + for batch in batches { + if is_cancelled(&request.import_id).await { + emit_progress( + &app, + TableImportProgress { + import_id: request.import_id.clone(), + status: TableImportStatus::Cancelled, + rows_imported, + total_rows, + error: None, + }, + ); + clear_cancelled(&request.import_id).await; + return Err("Import cancelled".to_string()); + } + + if let Err(error) = execute_on_pool(&state, &pool_key, &batch.sql).await { + emit_progress( + &app, + TableImportProgress { + import_id: request.import_id.clone(), + status: TableImportStatus::Error, + rows_imported, + total_rows, + error: Some(error.clone()), + }, + ); + return Err(error); + } + rows_imported = (rows_imported + batch.row_count).min(total_rows); + emit_progress( + &app, + TableImportProgress { + import_id: request.import_id.clone(), + status: TableImportStatus::Running, + rows_imported, + total_rows, + error: None, + }, + ); + } + + emit_progress( + &app, + TableImportProgress { + import_id: request.import_id.clone(), + status: TableImportStatus::Done, + rows_imported, + total_rows, + error: None, + }, + ); + clear_cancelled(&request.import_id).await; + + Ok(TableImportSummary { + import_id: request.import_id, + rows_imported, + total_rows, + }) +} + +#[tauri::command] +pub async fn cancel_table_import(import_id: String) -> Result { + CANCELLED_IMPORTS.write().await.insert(import_id); + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::models::connection::DatabaseType; + + #[test] + fn parses_csv_headers_and_preview_rows() { + let parsed = parse_csv_bytes(b"id,name,active\n1,Ada,true\n2,,false\n", 10).unwrap(); + + assert_eq!(parsed.columns, vec!["id", "name", "active"]); + assert_eq!(parsed.total_rows, 2); + assert_eq!( + parsed.rows[0], + vec![ + serde_json::Value::String("1".to_string()), + serde_json::Value::String("Ada".to_string()), + serde_json::Value::String("true".to_string()), + ] + ); + assert_eq!( + parsed.rows[1], + vec![ + serde_json::Value::String("2".to_string()), + serde_json::Value::Null, + serde_json::Value::String("false".to_string()), + ] + ); + } + + #[test] + fn parses_tsv_with_tab_delimiter() { + let parsed = parse_delimited_bytes(b"id\tname\n1\tAda\n", b'\t', 10).unwrap(); + + assert_eq!(parsed.columns, vec!["id", "name"]); + assert_eq!(parsed.total_rows, 1); + assert_eq!( + parsed.rows[0], + vec![ + serde_json::Value::String("1".to_string()), + serde_json::Value::String("Ada".to_string()), + ] + ); + } + + #[test] + fn parses_json_array_objects_with_union_columns() { + let parsed = + parse_json_bytes(br#"[{"id":1,"name":"Ada"},{"id":2,"active":true}]"#, 10).unwrap(); + + assert_eq!(parsed.columns, vec!["id", "name", "active"]); + assert_eq!(parsed.total_rows, 2); + assert_eq!( + parsed.rows[0], + vec![ + serde_json::json!(1), + serde_json::json!("Ada"), + serde_json::Value::Null, + ] + ); + assert_eq!( + parsed.rows[1], + vec![ + serde_json::json!(2), + serde_json::Value::Null, + serde_json::json!(true), + ] + ); + } + + #[test] + fn builds_import_insert_batches_from_mapped_columns() { + let mappings = vec![ + TableImportColumnMapping { + source_column: "id".to_string(), + target_column: "user_id".to_string(), + }, + TableImportColumnMapping { + source_column: "name".to_string(), + target_column: "display_name".to_string(), + }, + ]; + let data = ParsedImportFile { + columns: vec!["id".to_string(), "name".to_string(), "ignored".to_string()], + rows: vec![ + vec![ + serde_json::json!(1), + serde_json::json!("Ada"), + serde_json::json!("x"), + ], + vec![ + serde_json::json!(2), + serde_json::json!("O'Hara"), + serde_json::json!("y"), + ], + vec![ + serde_json::json!(3), + serde_json::Value::Null, + serde_json::json!("z"), + ], + ], + total_rows: 3, + }; + + let batches = build_import_insert_batches( + &data, + &mappings, + "users", + "public", + &DatabaseType::Postgres, + 2, + ) + .unwrap(); + + assert_eq!(batches, vec![ + ImportSqlBatch { + sql: "INSERT INTO \"public\".\"users\" (\"user_id\", \"display_name\") VALUES\n(1, 'Ada'),\n(2, 'O''Hara')".to_string(), + row_count: 2, + }, + ImportSqlBatch { + sql: "INSERT INTO \"public\".\"users\" (\"user_id\", \"display_name\") VALUES\n(3, NULL)".to_string(), + row_count: 1, + }, + ]); + } +} diff --git a/src-tauri/src/commands/transfer.rs b/src-tauri/src/commands/transfer.rs index 8eb340208..5c03cdd1a 100644 --- a/src-tauri/src/commands/transfer.rs +++ b/src-tauri/src/commands/transfer.rs @@ -50,7 +50,7 @@ pub enum TransferStatus { Cancelled, } -fn quote_identifier(name: &str, db_type: &DatabaseType) -> String { +pub(crate) fn quote_identifier(name: &str, db_type: &DatabaseType) -> String { match db_type { DatabaseType::Mysql | DatabaseType::ClickHouse | DatabaseType::Doris | DatabaseType::StarRocks => format!("`{}`", name.replace('`', "``")), DatabaseType::SqlServer => format!("[{}]", name.replace(']', "]]")), @@ -58,7 +58,7 @@ fn quote_identifier(name: &str, db_type: &DatabaseType) -> String { } } -fn qualified_table(table: &str, schema: &str, db_type: &DatabaseType) -> String { +pub(crate) fn qualified_table(table: &str, schema: &str, db_type: &DatabaseType) -> String { let qt = quote_identifier(table, db_type); if schema.is_empty() { qt @@ -251,7 +251,7 @@ fn generate_create_table_ddl( ddl } -fn generate_insert( +pub(crate) fn generate_insert( columns: &[String], rows: &[Vec], table: &str, @@ -312,7 +312,7 @@ fn count_sql(table: &str, schema: &str, db_type: &DatabaseType) -> String { format!("SELECT COUNT(*) FROM {full_table}") } -async fn execute_on_pool( +pub(crate) async fn execute_on_pool( state: &AppState, pool_key: &str, sql: &str, @@ -399,7 +399,7 @@ async fn execute_on_pool( } } -async fn get_db_type(state: &AppState, connection_id: &str) -> Result { +pub(crate) async fn get_db_type(state: &AppState, connection_id: &str) -> Result { let configs = state.configs.lock().await; configs .get(connection_id) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 9455f546e..8f9e6be76 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -64,6 +64,9 @@ pub fn run() { commands::sql_file::preview_sql_file, commands::sql_file::execute_sql_file, commands::sql_file::cancel_sql_file_execution, + commands::table_import::preview_table_import_file, + commands::table_import::import_table_file, + commands::table_import::cancel_table_import, commands::redis_cmd::redis_list_databases, commands::redis_cmd::redis_scan_keys, commands::redis_cmd::redis_get_value, diff --git a/src/App.vue b/src/App.vue index 60e3da68d..c97c8257c 100644 --- a/src/App.vue +++ b/src/App.vue @@ -1,7 +1,7 @@ + + diff --git a/src/components/sidebar/TreeItem.vue b/src/components/sidebar/TreeItem.vue index f0207cff3..f81a4c835 100644 --- a/src/components/sidebar/TreeItem.vue +++ b/src/components/sidebar/TreeItem.vue @@ -5,7 +5,7 @@ import { Database, Table, Columns3, Eye, ChevronRight, ChevronDown, Loader2, FolderOpen, Trash2, TerminalSquare, RefreshCw, Copy, TableProperties, Key, Link, Zap, ListTree, Pencil, Plug, Unplug, - Pin, ArrowRightLeft, Download, FileCode, Network, + Pin, ArrowRightLeft, Download, FileCode, Network, FileUp, } from "lucide-vue-next"; import { ContextMenu, ContextMenuContent, ContextMenuItem, @@ -43,6 +43,7 @@ const props = defineProps<{ const sqlFileUnsupportedTypes = new Set(["redis", "mongodb", "elasticsearch"]); const diagramSupportedTypes = new Set(["mysql", "postgres", "sqlite", "sqlserver", "oracle", "redshift"]); +const tableImportSupportedTypes = new Set(["mysql", "postgres", "sqlite", "duckdb", "clickhouse", "sqlserver", "oracle", "doris", "starrocks", "redshift"]); const isExportingDatabase = ref(false); function currentDatabaseType(): DatabaseType | undefined { @@ -513,6 +514,17 @@ function openDiagram() { }; } +function openTableImport() { + const node = props.node; + if (node.type !== "table" || !node.connectionId || !node.database) return; + connectionStore.tableImportSource = { + connectionId: node.connectionId, + database: node.database, + schema: node.schema, + tableName: node.label, + }; +} + const canExpand = !leafTypes.has(props.node.type); const canPin = computed(() => pinnableTypes.has(props.node.type)); const canOpenSqlFileExecution = computed(() => { @@ -523,6 +535,10 @@ const canOpenDiagram = computed(() => { const config = props.node.connectionId ? connectionStore.getConfig(props.node.connectionId) : undefined; return !!props.node.database && !!config && diagramSupportedTypes.has(config.db_type); }); +const canOpenTableImport = computed(() => { + const config = props.node.connectionId ? connectionStore.getConfig(props.node.connectionId) : undefined; + return props.node.type === "table" && !!props.node.database && !!config && tableImportSupportedTypes.has(config.db_type); +}); const isPinned = computed(() => props.node.pinned || connectionStore.isTreeNodePinned(props.node.id)); const hasTypeMenu = computed(() => { const t = props.node.type; @@ -685,6 +701,9 @@ async function showMore() { {{ t('diagram.open') }} + + {{ t('contextMenu.importData') }} + diff --git a/src/i18n/locales/en.ts b/src/i18n/locales/en.ts index f26304375..b6e7e74f2 100644 --- a/src/i18n/locales/en.ts +++ b/src/i18n/locales/en.ts @@ -226,6 +226,7 @@ export default { exportDatabase: "Export Database", exportDatabaseSuccess: "Exported {count} tables, up to {limit} rows each", exportDatabaseFailed: "Failed to export database: {message}", + importData: "Import Data", exportData: "Export Data", exportStructure: "Export Structure", }, @@ -349,6 +350,23 @@ export default { overallProgress: "Overall progress", dataTransfer: "Data Transfer", }, + tableImport: { + title: "Import Table Data", + target: "Target table", + selectFile: "Select File", + file: "File", + rows: "Rows", + mapped: "Mapped", + mapping: "Column Mapping", + skipColumn: "Skip", + preview: "Preview", + mode: "Import mode", + append: "Append data", + truncate: "Truncate then import", + progress: "Progress", + start: "Start Import", + success: "Imported {count} rows", + }, sqlFile: { title: "Execute SQL File", file: "File", diff --git a/src/i18n/locales/zh-CN.ts b/src/i18n/locales/zh-CN.ts index 8b995fd2c..e7c0ae1fd 100644 --- a/src/i18n/locales/zh-CN.ts +++ b/src/i18n/locales/zh-CN.ts @@ -226,6 +226,7 @@ export default { exportDatabase: "导出数据库", exportDatabaseSuccess: "已导出 {count} 张表,每表最多 {limit} 行", exportDatabaseFailed: "导出数据库失败:{message}", + importData: "导入数据", exportData: "导出数据", exportStructure: "导出表结构", }, @@ -349,6 +350,23 @@ export default { overallProgress: "整体进度", dataTransfer: "数据传输", }, + tableImport: { + title: "导入表数据", + target: "目标表", + selectFile: "选择文件", + file: "文件", + rows: "行数", + mapped: "已映射", + mapping: "字段映射", + skipColumn: "跳过", + preview: "预览", + mode: "导入模式", + append: "追加数据", + truncate: "清空后导入", + progress: "进度", + start: "开始导入", + success: "已导入 {count} 行", + }, sqlFile: { title: "执行 SQL 文件", file: "文件", diff --git a/src/lib/tableImport.ts b/src/lib/tableImport.ts new file mode 100644 index 000000000..2416d15ec --- /dev/null +++ b/src/lib/tableImport.ts @@ -0,0 +1,22 @@ +export const IMPORT_SKIP_TARGET = ""; + +export function normalizeImportColumnName(name: string): string { + return name.trim().toLowerCase().replace(/[_-]+/g, " ").replace(/\s+/g, " "); +} + +export function autoMapImportColumns( + sourceColumns: string[], + targetColumns: string[], +): Record { + const exactTargets = new Map(targetColumns.map((column) => [column, column])); + const normalizedTargets = new Map( + targetColumns.map((column) => [normalizeImportColumnName(column), column]), + ); + + return Object.fromEntries( + sourceColumns.map((source) => [ + source, + exactTargets.get(source) ?? normalizedTargets.get(normalizeImportColumnName(source)) ?? IMPORT_SKIP_TARGET, + ]), + ); +} diff --git a/src/lib/tauri.ts b/src/lib/tauri.ts index 171e5ef0b..2337afe12 100644 --- a/src/lib/tauri.ts +++ b/src/lib/tauri.ts @@ -409,3 +409,76 @@ export async function startTransfer( export async function cancelTransfer(transferId: string): Promise { return invoke("cancel_transfer", { transferId }); } + +// --- Table File Import --- +export type TableImportMode = "append" | "truncate"; +export type TableImportStatus = "running" | "done" | "error" | "cancelled"; + +export interface TableImportColumnMapping { + sourceColumn: string; + targetColumn: string; +} + +export interface TableImportPreview { + fileName: string; + filePath: string; + fileType: string; + sizeBytes: number; + columns: string[]; + rows: unknown[][]; + totalRows: number; +} + +export interface TableImportRequest { + importId: string; + connectionId: string; + database: string; + schema: string; + table: string; + filePath: string; + mappings: TableImportColumnMapping[]; + mode: TableImportMode; + batchSize: number; +} + +export interface TableImportSummary { + importId: string; + rowsImported: number; + totalRows: number; +} + +export interface TableImportProgress { + importId: string; + status: TableImportStatus; + rowsImported: number; + totalRows: number; + error?: string | null; +} + +export async function previewTableImportFile(filePath: string): Promise { + return invoke("preview_table_import_file", { filePath }); +} + +export async function importTableFile( + request: TableImportRequest, + onProgress: (progress: TableImportProgress) => void, +): Promise { + const unlisten: UnlistenFn = await listen("table-import-progress", (event) => { + if (event.payload.importId === request.importId) { + onProgress(event.payload); + if (event.payload.status === "done" || event.payload.status === "error" || event.payload.status === "cancelled") { + unlisten(); + } + } + }); + try { + return await invoke("import_table_file", { request }); + } catch (e) { + unlisten(); + throw e; + } +} + +export async function cancelTableImport(importId: string): Promise { + return invoke("cancel_table_import", { importId }); +} diff --git a/src/stores/connectionStore.ts b/src/stores/connectionStore.ts index 1b41d5036..9da7ec1fb 100644 --- a/src/stores/connectionStore.ts +++ b/src/stores/connectionStore.ts @@ -20,6 +20,7 @@ export const useConnectionStore = defineStore("connection", () => { const schemaDiffSource = ref<{ connectionId: string; database: string } | null>(null); const sqlFileSource = ref<{ connectionId: string; database: string } | null>(null); const diagramSource = ref<{ connectionId: string; database: string; schema?: string; tableName?: string } | null>(null); + const tableImportSource = ref<{ connectionId: string; database: string; schema?: string; tableName: string } | null>(null); function startEditing(id: string) { editingConnectionId.value = id; @@ -657,5 +658,6 @@ export const useConnectionStore = defineStore("connection", () => { schemaDiffSource, sqlFileSource, diagramSource, + tableImportSource, }; }); diff --git a/tests/tableImport.test.ts b/tests/tableImport.test.ts new file mode 100644 index 000000000..b3f4b1c89 --- /dev/null +++ b/tests/tableImport.test.ts @@ -0,0 +1,26 @@ +import { strict as assert } from "node:assert"; +import test from "node:test"; +import { + autoMapImportColumns, + normalizeImportColumnName, +} from "../src/lib/tableImport.ts"; + +test("normalizes import column names for matching", () => { + assert.equal(normalizeImportColumnName(" User ID "), "user id"); + assert.equal(normalizeImportColumnName("user_id"), "user id"); + assert.equal(normalizeImportColumnName("USER-ID"), "user id"); +}); + +test("auto maps source columns to matching target columns and skips unknown columns", () => { + const mapping = autoMapImportColumns( + ["id", "Name", "created-at", "ignored"], + ["user_id", "name", "created_at"], + ); + + assert.deepEqual(mapping, { + id: "", + Name: "name", + "created-at": "created_at", + ignored: "", + }); +});