xiaoxue_admin
/
dbx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(transfer): skip MySQL generated columns in inserts

This commit is contained in:
t8y2 2026-07-30 23:58:49 +08:00
parent 38e1c66f26
commit 2f24a4c6f6
No known key found for this signature in database
3 changed files with 293 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
crates/dbx-core/src/database_export.rs
View File

@ -10,8 +10,9 @@ use crate::mysql_ddl_normalize::DdlNormalizeOptions;
use crate::object_source_sql::build_export_object_source_sql;
use crate::sql_dialect::{qualified_table_name, quote_table_identifier, uses_single_row_insert_statements};
use crate::transfer::{
format_ch_array_sql_literal, format_pg_array_sql_literal, is_identity_column_extra, quote_identifier,
quote_postgres_string_literal, selected_columns_include_identity_extras, wrap_dameng_identity_insert_sql,
format_ch_array_sql_literal, format_pg_array_sql_literal, is_identity_column_extra,
is_mysql_generated_column_extra, quote_identifier, quote_postgres_string_literal,
selected_columns_include_identity_extras, wrap_dameng_identity_insert_sql,
wrap_dameng_identity_insert_sql_for_table,
};
use crate::types::ObjectSourceKind;
@ -796,11 +797,12 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
.iter()
.enumerate()
.filter(|(index, column)| {
!is_internal_export_column(options.database_type, column)
&& !is_postgres_tsvector_export_column(
options.database_type,
options.column_types.get(*index).and_then(|value| value.as_deref()),
)
is_export_insert_column(
options.database_type,
column,
options.column_types.get(*index).and_then(|value| value.as_deref()),
options.column_extras.get(*index).and_then(|value| value.as_deref()),
)
})
.collect::<Vec<_>>();
if insert_columns.is_empty() {
@ -870,6 +872,17 @@ fn is_postgres_tsvector_export_column(database_type: Option<DatabaseType>, colum
.unwrap_or(false)
}
fn is_export_insert_column(
database_type: Option<DatabaseType>,
column: &str,
column_type: Option<&str>,
column_extra: Option<&str>,
) -> bool {
!is_internal_export_column(database_type, column)
&& !is_postgres_tsvector_export_column(database_type, column_type)
&& !(database_type == Some(DatabaseType::Mysql) && is_mysql_generated_column_extra(column_extra))
}
fn is_postgres_json_export_column(database_type: Option<DatabaseType>, column_type: Option<&str>) -> bool {
database_type == Some(DatabaseType::Postgres)
&& column_type
@ -1283,9 +1296,59 @@ fn write_database_export_rows(
schema: &str,
db_type: &DatabaseType,
) -> Result<(), String> {
let mut insert_sql =
crate::transfer::generate_insert_typed(columns, column_types, rows, table, schema, db_type, None);
if *db_type == DatabaseType::Dameng && selected_columns_include_identity_extras(columns, column_extras) {
let insert_indices = columns
.iter()
.enumerate()
.filter(|(index, column)| {
is_export_insert_column(
Some(*db_type),
column,
column_types.get(*index).and_then(|value| value.as_deref()),
column_extras.get(*index).and_then(|value| value.as_deref()),
)
})
.map(|(index, _)| index)
.collect::<Vec<_>>();
if insert_indices.is_empty() {
return Ok(());
}
let filtered_columns;
let filtered_column_types;
let filtered_column_extras;
let filtered_rows;
let (insert_columns, insert_column_types, insert_column_extras, insert_rows) = if insert_indices.len()
== columns.len()
{
(columns, column_types, column_extras, rows)
} else {
filtered_columns = insert_indices.iter().map(|index| columns[*index].clone()).collect::<Vec<_>>();
filtered_column_types =
insert_indices.iter().map(|index| column_types.get(*index).cloned().unwrap_or(None)).collect::<Vec<_>>();
filtered_column_extras =
insert_indices.iter().map(|index| column_extras.get(*index).cloned().unwrap_or(None)).collect::<Vec<_>>();
filtered_rows = rows
.iter()
.map(|row| insert_indices.iter().map(|index| row.get(*index).cloned().unwrap_or(Value::Null)).collect())
.collect::<Vec<Vec<Value>>>();
(
filtered_columns.as_slice(),
filtered_column_types.as_slice(),
filtered_column_extras.as_slice(),
filtered_rows.as_slice(),
)
};
let mut insert_sql = crate::transfer::generate_insert_typed(
insert_columns,
insert_column_types,
insert_rows,
table,
schema,
db_type,
None,
);
if *db_type == DatabaseType::Dameng
&& selected_columns_include_identity_extras(insert_columns, insert_column_extras)
{
insert_sql = wrap_dameng_identity_insert_sql(&insert_sql, table, schema);
}
if insert_sql.is_empty() {
@ -2172,10 +2235,10 @@ mod tests {
format_export_table_ddl, generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl,
generate_postgres_sequence_owner_ddl, generate_postgres_sequence_setval_sql,
is_postgres_extension_member_routine, mysql_database_export_preamble, normalize_export_table_ddl,
record_export_error, BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions,
DatabaseExportObjectCounts, DatabaseExportRequest, DdlNormalizeOptions, ExportedTableSql,
PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers, DATABASE_EXPORT_INSERT_BATCH_SIZE,
DATABASE_EXPORT_ROW_LIMIT,
record_export_error, write_database_export_rows, BuildDatabaseSqlExportOptions,
BuildExportInsertStatementsOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DdlNormalizeOptions,
ExportedTableSql, PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers,
DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT,
};
use crate::models::connection::DatabaseType;
use crate::types::{ObjectInfo, ObjectSourceKind, TableInfo};
@ -2965,6 +3028,97 @@ mod tests {
assert_eq!(statements, vec!["INSERT INTO \"public\".\"articles\" (\"id\", \"title\") VALUES (1, 'Hello');"]);
}
#[test]
fn mysql_generated_columns_are_omitted_from_sql_inserts_but_kept_in_ddl() {
let ddl = "CREATE TABLE `orders` (`id` bigint AUTO_INCREMENT, `quantity` int, `unit_price` decimal(10,2), `virtual_total` decimal(10,2) GENERATED ALWAYS AS ((`quantity` * `unit_price`)) VIRTUAL, `stored_total` decimal(10,2) GENERATED ALWAYS AS ((`quantity` * `unit_price`)) STORED);";
let sql = build_database_sql_export(BuildDatabaseSqlExportOptions {
database_name: "shop".to_string(),
exported_at: Some("2026-07-30T00:00:00.000Z".to_string()),
tables: vec![ExportedTableSql {
display_name: "orders".to_string(),
database_type: Some(DatabaseType::Mysql),
schema: None,
table_name: Some("orders".to_string()),
qualified_table_name: None,
ddl: Some(ddl.to_string()),
columns: vec![
"id".to_string(),
"quantity".to_string(),
"unit_price".to_string(),
"virtual_total".to_string(),
"stored_total".to_string(),
"created_at".to_string(),
],
column_types: vec![
Some("bigint".to_string()),
Some("int".to_string()),
Some("decimal(10,2)".to_string()),
Some("decimal(10,2)".to_string()),
Some("decimal(10,2)".to_string()),
Some("timestamp".to_string()),
],
column_extras: vec![
Some("auto_increment".to_string()),
None,
None,
Some("VIRTUAL GENERATED".to_string()),
Some("stored generated".to_string()),
Some("DEFAULT_GENERATED".to_string()),
],
rows: vec![vec![json!(7), json!(2), json!(3.5), json!(7.0), json!(7.0), json!("2026-07-30 08:00:00")]],
truncated: false,
}],
row_limit_per_table: Some(DATABASE_EXPORT_ROW_LIMIT),
insert_batch_size: Some(DATABASE_EXPORT_INSERT_BATCH_SIZE),
connection_id: None,
database: None,
schema: None,
omit_auto_increment: false,
})
.unwrap();
assert!(sql.contains(ddl));
assert!(sql.contains(
"INSERT INTO `orders` (`id`, `quantity`, `unit_price`, `created_at`) VALUES (7, 2, 3.5, '2026-07-30 08:00:00');"
));
assert!(!sql.contains("INSERT INTO `orders` (`id`, `quantity`, `unit_price`, `virtual_total`"));
}
#[test]
fn mysql_database_export_file_rows_omit_generated_columns() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("orders.sql");
let mut file = std::fs::File::create(&path).unwrap();
write_database_export_rows(
&mut file,
&[vec![json!(7), json!(2), json!(7.0), json!("2026-07-30 08:00:00")]],
&["id".to_string(), "quantity".to_string(), "virtual_total".to_string(), "created_at".to_string()],
&[
Some("bigint".to_string()),
Some("int".to_string()),
Some("decimal(10,2)".to_string()),
Some("timestamp".to_string()),
],
&[
Some("auto_increment".to_string()),
None,
Some("VIRTUAL GENERATED".to_string()),
Some("DEFAULT_GENERATED".to_string()),
],
"orders",
"shop",
&DatabaseType::Mysql,
)
.unwrap();
drop(file);
assert_eq!(
std::fs::read_to_string(path).unwrap(),
"INSERT INTO `orders` (`id`, `quantity`, `created_at`) VALUES\n(7, 2, '2026-07-30 08:00:00');\n\n"
);
}
#[test]
fn dameng_identity_export_inserts_enable_identity_insert() {
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {

61
crates/dbx-core/src/table_export.rs
View File

@ -138,6 +138,25 @@ fn resolve_requested_export_columns(
(resolved_columns, resolved_column_types, resolved_primary_keys)
}
fn requested_export_needs_column_extras(database_type: DatabaseType, format: &str) -> bool {
database_type == DatabaseType::Mysql && format.eq_ignore_ascii_case("sql")
}
fn resolve_requested_export_column_extras(
requested_columns: &[String],
table_columns: &[crate::db::ColumnInfo],
) -> Vec<Option<String>> {
requested_columns
.iter()
.map(|requested| {
table_columns
.iter()
.find(|column| column.name.eq_ignore_ascii_case(requested))
.and_then(|column| column.extra.clone())
})
.collect()
}
fn write_json_row_object<W: Write>(writer: &mut W, columns: &[String], row: &[Value]) -> Result<(), String> {
writer.write_all(b"{\n").map_err(|e| format!("Failed to write JSON: {e}"))?;
let mut first = true;
@ -1107,7 +1126,20 @@ async fn export_table_data_core_inner(
request.column_types.as_deref(),
request.primary_keys.as_deref(),
);
(col_names, column_types, Vec::new(), primary_keys)
let column_extras = if requested_export_needs_column_extras(db_type, &request.format) {
let table_columns = crate::schema::get_columns_core(
state,
&request.connection_id,
&request.database,
request.schema.as_deref().unwrap_or(""),
&request.table_name,
)
.await?;
resolve_requested_export_column_extras(&col_names, &table_columns)
} else {
Vec::new()
};
(col_names, column_types, column_extras, primary_keys)
} else {
let columns = crate::schema::get_columns_core(
state,
@ -2131,6 +2163,33 @@ mod tests {
assert_eq!(resolved_mysql, mysql_columns);
}
#[test]
fn requested_mysql_sql_export_resolves_generated_column_extras_only_for_sql() {
let table_columns = vec![
crate::db::ColumnInfo {
name: "ID".to_string(),
extra: Some("auto_increment".to_string()),
..Default::default()
},
crate::db::ColumnInfo {
name: "virtual_total".to_string(),
extra: Some("VIRTUAL GENERATED".to_string()),
..Default::default()
},
];
let requested_columns = vec!["virtual_total".to_string(), "id".to_string(), "missing".to_string()];
assert!(requested_export_needs_column_extras(DatabaseType::Mysql, "SQL"));
for format in ["csv", "json", "xlsx"] {
assert!(!requested_export_needs_column_extras(DatabaseType::Mysql, format));
}
assert!(!requested_export_needs_column_extras(DatabaseType::Postgres, "sql"));
assert_eq!(
resolve_requested_export_column_extras(&requested_columns, &table_columns),
vec![Some("VIRTUAL GENERATED".to_string()), Some("auto_increment".to_string()), None]
);
}
#[test]
fn agent_table_read_unsupported_detects_old_agent_errors() {
assert!(is_agent_table_read_unsupported("Agent RPC error (-1): unknown method: start_table_read"));

66
crates/dbx-core/src/transfer.rs
View File

@ -495,6 +495,20 @@ pub(crate) fn is_identity_column_extra(extra: Option<&str>) -> bool {
})
}
pub(crate) fn is_mysql_generated_column_extra(extra: Option<&str>) -> bool {
extra.is_some_and(|value| {
let mut parts = value.split_whitespace();
let Some(first) = parts.next() else {
return false;
};
if first.eq_ignore_ascii_case("generated") {
return true;
}
matches!(first.to_ascii_lowercase().as_str(), "virtual" | "stored" | "persistent")
&& parts.next().is_some_and(|part| part.eq_ignore_ascii_case("generated"))
})
}
pub(crate) fn selected_columns_include_identity_extras(columns: &[String], column_extras: &[Option<String>]) -> bool {
columns
.iter()
@ -523,6 +537,10 @@ fn is_sqlserver_non_insertable_transfer_column(
&& is_sqlserver_rowversion_type(&column.data_type)
}
fn is_mysql_non_insertable_transfer_column(column: &db::ColumnInfo, source_db_type: &DatabaseType) -> bool {
*source_db_type == DatabaseType::Mysql && is_mysql_generated_column_extra(column.extra.as_deref())
}
fn writable_transfer_columns(
columns: &[db::ColumnInfo],
source_db_type: &DatabaseType,
@ -530,7 +548,10 @@ fn writable_transfer_columns(
) -> Vec<db::ColumnInfo> {
columns
.iter()
.filter(|column| !is_sqlserver_non_insertable_transfer_column(column, source_db_type, target_db_type))
.filter(|column| {
!is_sqlserver_non_insertable_transfer_column(column, source_db_type, target_db_type)
&& !is_mysql_non_insertable_transfer_column(column, source_db_type)
})
.cloned()
.collect()
}
@ -5342,6 +5363,49 @@ mod tests {
assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::<Vec<_>>(), vec!["id", "name"]);
}
#[test]
fn mysql_writable_transfer_columns_skip_only_generated_columns() {
let columns = vec![
test_column("id", "int"),
db::ColumnInfo { extra: Some("DEFAULT_GENERATED".to_string()), ..test_column("created_at", "timestamp") },
db::ColumnInfo { extra: Some("auto_increment".to_string()), ..test_column("sequence_id", "bigint") },
db::ColumnInfo {
extra: Some("VIRTUAL GENERATED".to_string()),
..test_column("virtual_total", "decimal(10,2)")
},
db::ColumnInfo { extra: Some("stored generated".to_string()), ..test_column("stored_hash", "varchar(64)") },
db::ColumnInfo {
extra: Some("PERSISTENT GENERATED".to_string()),
..test_column("persistent_total", "decimal(10,2)")
},
db::ColumnInfo { extra: Some("GENERATED ALWAYS".to_string()), ..test_column("explicit_generated", "int") },
db::ColumnInfo {
extra: Some("on update CURRENT_TIMESTAMP".to_string()),
..test_column("updated_at", "timestamp")
},
];
let writable = writable_transfer_columns(&columns, &DatabaseType::Mysql, &DatabaseType::Mysql);
assert_eq!(
writable.iter().map(|column| column.name.as_str()).collect::<Vec<_>>(),
vec!["id", "created_at", "sequence_id", "updated_at"]
);
assert_eq!(columns.len(), 8, "DDL metadata must retain generated columns");
}
#[test]
fn non_mysql_transfer_columns_keep_generated_markers() {
let columns = vec![
test_column("id", "int"),
db::ColumnInfo { extra: Some("STORED GENERATED".to_string()), ..test_column("computed", "int") },
];
let writable = writable_transfer_columns(&columns, &DatabaseType::Postgres, &DatabaseType::Postgres);
assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::<Vec<_>>(), vec!["id", "computed"]);
}
#[test]
fn non_sqlserver_target_writable_transfer_columns_keep_timestamp_type() {
let columns = vec![test_column("id", "int"), test_column("updated_at", "timestamp")];