From 0f312084ffb060c91c44d240949e6cc364db8b72 Mon Sep 17 00:00:00 2001 From: zipg Date: Thu, 25 Jun 2026 23:54:09 +0800 Subject: [PATCH] fix(sidebar): paginate table list by page size --- .../editor/EditorSettingsDialog.vue | 3 + .../src/components/sidebar/ConnectionTree.vue | 38 +++- apps/desktop/src/stores/connectionStore.ts | 158 +++++++++++++++-- crates/dbx-core/src/db/mysql.rs | 166 +++++++++++++++++- crates/dbx-core/src/schema.rs | 19 +- 5 files changed, 346 insertions(+), 38 deletions(-) diff --git a/apps/desktop/src/components/editor/EditorSettingsDialog.vue b/apps/desktop/src/components/editor/EditorSettingsDialog.vue index 67b526016..ba13d4c76 100644 --- a/apps/desktop/src/components/editor/EditorSettingsDialog.vue +++ b/apps/desktop/src/components/editor/EditorSettingsDialog.vue @@ -564,6 +564,7 @@ function hasChanges(): boolean { async function persistSettings() { if (hasApplyBlocker.value) return; const sidebarObjectDisplayChanged = editSidebarObjectDisplay.value !== settingsStore.editorSettings.sidebarObjectDisplay; + const sidebarTablePageSizeChanged = editSidebarTablePageSize.value !== (settingsStore.desktopSettings.sidebar_table_page_size ?? DEFAULT_SIDEBAR_TABLE_PAGE_SIZE); settingsStore.updateEditorSettings({ fontFamily: editFontFamily.value, fontSize: editFontSize.value, @@ -613,6 +614,8 @@ async function persistSettings() { desktopCloseBehaviorResetPending.value = false; if (sidebarObjectDisplayChanged) { await connectionStore.refreshAllTree(); + } else if (sidebarTablePageSizeChanged) { + await connectionStore.refreshSidebarObjectPagination(); } } diff --git a/apps/desktop/src/components/sidebar/ConnectionTree.vue b/apps/desktop/src/components/sidebar/ConnectionTree.vue index 792b402b5..1ba49f721 100644 --- a/apps/desktop/src/components/sidebar/ConnectionTree.vue +++ b/apps/desktop/src/components/sidebar/ConnectionTree.vue @@ -33,7 +33,7 @@ const plainTreeScrollerRef = ref(null); type SearchScope = "connection" | "database" | "schema" | "table" | "view"; const selectedSearchScopes = ref([]); const searchCollapsedIds = ref>(new Set()); -const searchRefreshedGroupIds = new Set(); +const searchRefreshedNodeIds = new Set(); let searchTimer: number | undefined; watch( @@ -58,30 +58,50 @@ watch(deferredSearchQuery, (newQuery, oldQuery) => { store.sidebarSearchQuery = newQuery; const tasks: Promise[] = []; for (const root of store.treeNodes) { - collectExpandedObjectGroups(root, tasks, newQuery ? searchRefreshedGroupIds : undefined); + collectExpandedObjectSearchTargets(root, tasks, newQuery ? searchRefreshedNodeIds : undefined); } if (!newQuery && oldQuery) { - searchRefreshedGroupIds.clear(); + searchRefreshedNodeIds.clear(); } Promise.all(tasks).catch(() => {}); }); const searchableObjectGroupTypes = new Set(["group-tables", "group-views", "group-materialized-views"]); +const simpleObjectParentTypes = new Set(["database", "schema", "linked-server-schema"]); +const simpleObjectChildTypes = new Set(["table", "view", "materialized_view", "procedure", "function", "sequence", "package", "package-body", "load-more"]); -function collectExpandedObjectGroups(node: TreeNode, tasks: Promise[], refreshedGroupIds?: Set) { - if (refreshedGroupIds && node.isExpanded && node.children) { +function isSimpleObjectSearchParent(node: TreeNode): boolean { + return ( + settingsStore.editorSettings.sidebarObjectDisplay === "simple" && + simpleObjectParentTypes.has(node.type) && + node.isExpanded === true && + !!node.children?.some((child) => simpleObjectChildTypes.has(child.type)) + ); +} + +function collectExpandedObjectSearchTargets(node: TreeNode, tasks: Promise[], refreshedNodeIds?: Set) { + if (refreshedNodeIds && isSimpleObjectSearchParent(node)) { + refreshedNodeIds.add(node.id); + tasks.push(store.refreshTreeNode(node)); + return; + } + if (refreshedNodeIds && node.isExpanded && node.children) { for (const child of node.children) { if (child.connectionId && searchableObjectGroupTypes.has(child.type)) { - refreshedGroupIds.add(child.id); + refreshedNodeIds.add(child.id); tasks.push(store.loadObjectGroupChildren(child, { force: true })); } } - } else if (!refreshedGroupIds && searchRefreshedGroupIds.has(node.id)) { - tasks.push(store.loadObjectGroupChildren(node, { force: true })); + } else if (!refreshedNodeIds && searchRefreshedNodeIds.has(node.id)) { + if (searchableObjectGroupTypes.has(node.type)) { + tasks.push(store.loadObjectGroupChildren(node, { force: true })); + } else if (simpleObjectParentTypes.has(node.type)) { + tasks.push(store.refreshTreeNode(node)); + } } if (node.children) { for (const child of node.children) { - collectExpandedObjectGroups(child, tasks, refreshedGroupIds); + collectExpandedObjectSearchTargets(child, tasks, refreshedNodeIds); } } } diff --git a/apps/desktop/src/stores/connectionStore.ts b/apps/desktop/src/stores/connectionStore.ts index 474a0b432..3e0bf9a65 100644 --- a/apps/desktop/src/stores/connectionStore.ts +++ b/apps/desktop/src/stores/connectionStore.ts @@ -641,9 +641,7 @@ export const useConnectionStore = defineStore("connection", () => { return { children: [], objectCount: 0, hasMore: false, nextOffset: options.offset }; } const searchFilter = sidebarSearchQuery.value || undefined; - // When searching, fetch all matching tables (no pagination) — backend filter - // already narrows the result set, so client-side filtering is not needed. - const fetchLimit = searchFilter ? undefined : options.pageSize + 1; + const fetchLimit = searchFilter ? options.pageSize : options.pageSize + 1; const tables = await api.listTables(options.node.connectionId, options.node.database, options.querySchema, searchFilter, fetchLimit, searchFilter ? undefined : options.offset, options.objectTypes); const hasMore = searchFilter ? false : tables.length > options.pageSize; const pageTables = hasMore ? tables.slice(0, options.pageSize) : tables; @@ -664,6 +662,59 @@ export const useConnectionStore = defineStore("connection", () => { }; } + async function loadPagedSimpleTableChildren(options: { + nodeId: string; + connectionId: string; + database: string; + querySchema: string; + effectiveSchema?: string; + nonTableObjectTypes: DatabaseObjectTreeKind[]; + offset: number; + pageSize: number; + }): Promise<{ children: TreeNode[]; objectCount: number; hasMore: boolean; nextOffset: number }> { + const searchFilter = sidebarSearchQuery.value || undefined; + const fetchLimit = searchFilter ? options.pageSize : options.pageSize + 1; + const tables = await api.listTables(options.connectionId, options.database, options.querySchema, searchFilter, fetchLimit, searchFilter ? undefined : options.offset); + const hasMore = searchFilter ? false : tables.length > options.pageSize; + const pageTables = hasMore ? tables.slice(0, options.pageSize) : tables; + indexCompletionTables(options.connectionId, options.database, options.effectiveSchema, tableInfosToCompletionTables(pageTables, options.effectiveSchema)); + + if (!searchFilter) { + try { + const objects = options.nonTableObjectTypes.length > 0 ? await api.listObjects(options.connectionId, options.database, options.querySchema, options.nonTableObjectTypes) : []; + const children = buildSimpleObjectTreeNodes({ + nodeId: options.nodeId, + connectionId: options.connectionId, + database: options.database, + schema: options.effectiveSchema, + objects: mergeTableInfosIntoObjects(objects, pageTables, options.effectiveSchema), + }); + return { + children, + objectCount: children.length, + hasMore, + nextOffset: options.offset + pageTables.length, + }; + } catch { + // Some drivers only expose table metadata; keep the paged table tree usable. + } + } + + const children = buildTableTreeNodes({ + nodeId: options.nodeId, + connectionId: options.connectionId, + database: options.database, + schema: options.effectiveSchema, + tables: pageTables, + }); + return { + children, + objectCount: children.length, + hasMore, + nextOffset: options.offset + pageTables.length, + }; + } + function refreshStaleTreeNode(node: TreeNode) { if (staleTreeRefreshIds.has(node.id)) return; staleTreeRefreshIds.add(node.id); @@ -1812,23 +1863,22 @@ export const useConnectionStore = defineStore("connection", () => { const config = getConfig(connectionId); const querySchema = connectionObjectTreeQuerySchema(config, database, schema); const effectiveSchema = connectionObjectTreeNodeSchema(config, database, schema); + const nonTableObjectTypes = simpleObjectDisplay ? supportedSidebarObjectTypes(config).filter((objectType) => objectType !== "TABLE") : []; let children: TreeNode[]; if (simpleObjectDisplay) { - try { - const [objects, tables] = await Promise.all([api.listObjects(connectionId, database, querySchema), api.listTables(connectionId, database, querySchema)]); - indexCompletionTables(connectionId, database, effectiveSchema, tableInfosToCompletionTables(tables, effectiveSchema)); - children = buildSimpleObjectTreeNodes({ - nodeId, - connectionId, - database, - schema: effectiveSchema, - objects: mergeTableInfosIntoObjects(objects, tables, effectiveSchema), - }); - } catch { - const tables = await api.listTables(connectionId, database, querySchema); - indexCompletionTables(connectionId, database, effectiveSchema, tableInfosToCompletionTables(tables, effectiveSchema)); - children = buildTableTreeNodes({ nodeId, connectionId, database, schema: effectiveSchema, tables }); - } + const pageSize = sidebarObjectGroupPageSize(); + const page = await loadPagedSimpleTableChildren({ + nodeId, + connectionId, + database, + querySchema, + effectiveSchema, + nonTableObjectTypes, + offset: 0, + pageSize, + }); + children = page.hasMore && !sidebarSearchQuery.value ? [...page.children, buildLoadMoreNode(node, page.nextOffset, pageSize)] : page.children; + node.objectCount = page.objectCount; } else { children = buildObjectGroupPlaceholderNodes({ nodeId, @@ -1916,6 +1966,31 @@ export const useConnectionStore = defineStore("connection", () => { node.isLoading = true; try { await ensureConnected(parent.connectionId); + if (parent.type === "database" || parent.type === "schema" || parent.type === "linked-server-schema") { + const parentDatabase = parent.database; + if (!parentDatabase) return; + const config = getConfig(parent.connectionId); + const querySchema = connectionObjectTreeQuerySchema(config, parentDatabase, parent.schema); + const effectiveSchema = connectionObjectTreeNodeSchema(config, parentDatabase, parent.schema); + const page = await loadPagedSimpleTableChildren({ + nodeId: parent.schema ? `${parent.connectionId}:${parentDatabase}:${parent.schema}` : `${parent.connectionId}:${parentDatabase}`, + connectionId: parent.connectionId, + database: parentDatabase, + querySchema, + effectiveSchema, + nonTableObjectTypes: [], + offset: node.loadMore.offset, + pageSize: node.loadMore.pageSize, + }); + const currentChildren = withoutLoadMoreNodes(parent.children); + const mergedChildren = mergeTableTreePageChildren(currentChildren, page.children, parent.connectionId, parentDatabase); + const nextChildren = page.hasMore ? [...mergedChildren, buildLoadMoreNode(parent, page.nextOffset, node.loadMore.pageSize)] : mergedChildren; + parent.objectCount = (parent.objectCount ?? currentChildren.length) + page.objectCount; + setChildren(parent, nextChildren); + await savePersistedTreeChildren(schemaCacheKey(parent.connectionId, parentDatabase, parent.schema || "", "objects-simple-v3"), nextChildren); + parent.isExpanded = true; + return; + } const objectTypes = objectTypesForGroupNode(parent.type); const parentNodeId = objectGroupRefreshParentId(parent); if (!objectTypes || !parentNodeId) return; @@ -3083,6 +3158,52 @@ export const useConnectionStore = defineStore("connection", () => { await refreshExpandedNodes(treeNodes.value); } + async function refreshSidebarObjectPagination() { + const simpleObjectDisplay = useSettingsStore().editorSettings.sidebarObjectDisplay === "simple"; + const isDirectObjectParent = (node: TreeNode) => { + if (!node.children || node.children.length === 0) return false; + return node.children.some( + (child) => child.type === "table" || child.type === "view" || child.type === "materialized_view" || child.type === "procedure" || child.type === "function" || child.type === "sequence" || child.type === "package" || child.type === "package-body" || child.type === "load-more", + ); + }; + const refreshNodes = async (nodes: TreeNode[]) => { + for (const node of nodes) { + if (node.type === "connection-group") { + if (node.children) await refreshNodes(node.children); + continue; + } + if (objectTypesForGroupNode(node.type)) { + if (node.connectionId && connectedIds.value.has(node.connectionId)) { + clearLoadedChildrenCache(node.id); + if (node.isExpanded) { + await loadObjectGroupChildren(node, { force: true }); + } else if (node.children) { + node.children = []; + } + } + continue; + } + if (simpleObjectDisplay && (node.type === "database" || node.type === "schema" || node.type === "linked-server-schema")) { + if (isDirectObjectParent(node)) { + if (node.connectionId && connectedIds.value.has(node.connectionId)) { + clearLoadedChildrenCache(node.id); + if (node.isExpanded) { + await refreshTreeNode(node); + } else { + node.children = []; + } + } + continue; + } + if (node.children) await refreshNodes(node.children); + continue; + } + if (node.children) await refreshNodes(node.children); + } + }; + await refreshNodes(treeNodes.value); + } + async function exportConnectionsToFile(passphrase: string) { const { encryptConfig } = await import("@/lib/configCrypto"); const exportData = { connections: connections.value, layout: sidebarLayout.value }; @@ -3449,6 +3570,7 @@ export const useConnectionStore = defineStore("connection", () => { treeNodes, removeTreeNode, refreshAllTree, + refreshSidebarObjectPagination, refreshTreeNode, refreshDatabaseTreeNode, refreshObjectListTreeNode, diff --git a/crates/dbx-core/src/db/mysql.rs b/crates/dbx-core/src/db/mysql.rs index 2c7b9d257..ad0177b78 100644 --- a/crates/dbx-core/src/db/mysql.rs +++ b/crates/dbx-core/src/db/mysql.rs @@ -1045,10 +1045,18 @@ fn database_infos_from_names( } pub async fn list_tables(pool: &MySqlPool, database: &str) -> Result, String> { - let sql = format!( - "SELECT TABLE_NAME, TABLE_TYPE, TABLE_COMMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = {} ORDER BY TABLE_NAME", - quote_value(database), - ); + list_tables_filtered(pool, database, None, None, None, None).await +} + +pub async fn list_tables_filtered( + pool: &MySqlPool, + database: &str, + filter: Option<&str>, + limit: Option, + offset: Option, + object_types: Option<&[String]>, +) -> Result, String> { + let sql = list_tables_sql(database, filter, limit, offset, object_types); let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?; let result = match conn.query_iter(&sql).await { Ok(result) => result, @@ -1056,7 +1064,9 @@ pub async fn list_tables(pool: &MySqlPool, database: &str) -> Result = result.collect_and_drop().await.map_err(|e| e.to_string())?; @@ -1077,7 +1087,7 @@ pub async fn list_tables(pool: &MySqlPool, database: &str) -> Result Result, + limit: Option, + offset: Option, + object_types: Option<&[String]>, +) -> bool { + let has_filter = filter.is_some_and(|filter| !filter.trim().is_empty()); + let has_object_types = object_types.is_some_and(|object_types| !object_types.is_empty()); + !has_filter && limit.is_none() && offset.unwrap_or(0) == 0 && !has_object_types +} + +fn filter_list_tables_fallback( + tables: Vec, + filter: Option<&str>, + limit: Option, + offset: Option, + object_types: Option<&[String]>, +) -> Vec { + let filter = filter.unwrap_or("").trim().to_ascii_lowercase(); + let normalized_object_types: Vec = object_types + .unwrap_or(&[]) + .iter() + .map(|object_type| object_type.to_ascii_uppercase().replace(' ', "_")) + .collect(); + let wants_table = + normalized_object_types.is_empty() || normalized_object_types.iter().any(|object_type| object_type == "TABLE"); + let wants_view = + normalized_object_types.is_empty() || normalized_object_types.iter().any(|object_type| object_type == "VIEW"); + + tables + .into_iter() + .filter(|table| filter.is_empty() || table.name.to_ascii_lowercase().contains(&filter)) + .filter(|table| if table.table_type.eq_ignore_ascii_case("VIEW") { wants_view } else { wants_table }) + .skip(offset.unwrap_or(0)) + .take(limit.unwrap_or(usize::MAX)) + .collect() +} + +fn list_tables_sql( + database: &str, + filter: Option<&str>, + limit: Option, + offset: Option, + object_types: Option<&[String]>, +) -> String { + let mut sql = format!( + "SELECT TABLE_NAME, TABLE_TYPE, TABLE_COMMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = {}", + quote_value(database), + ); + if let Some(object_types) = object_types.filter(|object_types| !object_types.is_empty()) { + let wants_table = object_types + .iter() + .map(|object_type| object_type.to_ascii_uppercase().replace(' ', "_")) + .any(|object_type| object_type == "TABLE"); + let wants_view = object_types + .iter() + .map(|object_type| object_type.to_ascii_uppercase().replace(' ', "_")) + .any(|object_type| object_type == "VIEW"); + match (wants_table, wants_view) { + (true, false) => sql.push_str(" AND TABLE_TYPE <> 'VIEW'"), + (false, true) => sql.push_str(" AND TABLE_TYPE = 'VIEW'"), + (false, false) => sql.push_str(" AND 1 = 0"), + (true, true) => {} + } + } + if let Some(filter) = filter.map(str::trim).filter(|filter| !filter.is_empty()) { + let escaped = filter.to_ascii_lowercase().replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_"); + let pattern = format!("%{}%", escaped); + sql.push_str(&format!(" AND LOWER(TABLE_NAME) LIKE {} ESCAPE '\\\\'", quote_value(&pattern))); + } + sql.push_str(" ORDER BY TABLE_NAME"); + if let Some(limit) = limit { + sql.push_str(&format!(" LIMIT {}", limit)); + } + if let Some(offset) = offset.filter(|offset| *offset > 0) { + sql.push_str(&format!(" OFFSET {}", offset)); + } + sql +} + pub async fn completion_assistant_search( pool: &MySqlPool, request: &CompletionAssistantRequest, @@ -2483,6 +2573,70 @@ mod tests { assert!(sql.contains("UPDATE_TIME")); } + #[test] + fn mysql_list_tables_sql_applies_filter_limit_and_offset() { + let sql = list_tables_sql("app", Some("user_%"), Some(101), Some(200), None); + + assert!(sql.contains("FROM information_schema.TABLES")); + assert!(sql.contains("TABLE_SCHEMA = 'app'")); + assert!(sql.contains("LOWER(TABLE_NAME) LIKE '%user\\\\_\\\\%%' ESCAPE '\\\\'")); + assert!(sql.contains("ORDER BY TABLE_NAME")); + assert!(sql.contains("LIMIT 101")); + assert!(sql.contains("OFFSET 200")); + } + + #[test] + fn mysql_list_tables_sql_filters_table_type_before_pagination() { + let tables = vec!["TABLE".to_string()]; + let table_sql = list_tables_sql("app", None, Some(1000), None, Some(&tables)); + assert!(table_sql.contains("TABLE_TYPE <> 'VIEW'")); + assert!(table_sql.find("TABLE_TYPE <> 'VIEW'") < table_sql.find("ORDER BY TABLE_NAME")); + assert!(table_sql.find("ORDER BY TABLE_NAME") < table_sql.find("LIMIT 1000")); + + let views = vec!["VIEW".to_string()]; + let view_sql = list_tables_sql("app", None, Some(1000), None, Some(&views)); + assert!(view_sql.contains("TABLE_TYPE = 'VIEW'")); + } + + #[test] + fn mysql_empty_list_tables_fallback_only_for_unfiltered_query() { + assert!(should_fallback_empty_list_tables(None, None, None, None)); + assert!(!should_fallback_empty_list_tables(Some("missing"), None, None, None)); + assert!(!should_fallback_empty_list_tables(None, Some(1000), None, None)); + assert!(!should_fallback_empty_list_tables(None, None, Some(1000), None)); + assert!(!should_fallback_empty_list_tables(None, None, None, Some(&["VIEW".to_string()]))); + } + + #[test] + fn mysql_show_tables_fallback_applies_filter_type_limit_and_offset() { + let rows = vec![ + TableInfo { + name: "audit_2024".to_string(), + table_type: "BASE TABLE".to_string(), + comment: None, + parent_schema: None, + parent_name: None, + }, + TableInfo { + name: "audit_view".to_string(), + table_type: "VIEW".to_string(), + comment: None, + parent_schema: None, + parent_name: None, + }, + TableInfo { + name: "audit_2025".to_string(), + table_type: "BASE TABLE".to_string(), + comment: None, + parent_schema: None, + parent_name: None, + }, + ]; + let filtered = filter_list_tables_fallback(rows, Some("audit"), Some(1), Some(1), Some(&["TABLE".to_string()])); + + assert_eq!(filtered.iter().map(|table| table.name.as_str()).collect::>(), vec!["audit_2025"]); + } + #[test] fn mysql_table_comment_sql_targets_single_table() { let sql = table_comment_sql("app", "users"); diff --git a/crates/dbx-core/src/schema.rs b/crates/dbx-core/src/schema.rs index 144a59a13..9873531e2 100644 --- a/crates/dbx-core/src/schema.rs +++ b/crates/dbx-core/src/schema.rs @@ -1011,12 +1011,21 @@ async fn list_tables_once( .map(|tables| filter_table_infos(tables, filter, limit, offset, object_types)) } PoolKind::Mysql(p, mode) => { - let tables = if *mode == MysqlMode::OceanBaseOracle { - db::ob_oracle::list_tables(p, schema).await + if *mode == MysqlMode::OceanBaseOracle { + let tables = db::ob_oracle::list_tables(p, schema).await?; + Ok(filter_table_infos(tables, filter, limit, offset, object_types)) } else { - db::mysql::list_tables(p, mysql_table_metadata_catalog(database, schema)).await - }?; - Ok(filter_table_infos(tables, filter, limit, offset, object_types)) + db::mysql::list_tables_filtered( + p, + mysql_table_metadata_catalog(database, schema), + filter, + limit, + offset, + object_types, + ) + .await + .map(|tables| filter_table_infos(tables, None, None, None, object_types)) + } } PoolKind::Postgres(p) if db_config.as_ref().is_some_and(is_questdb_config) => { db::questdb::list_tables(p, schema)