From fc9f97b6b57ede714d7c7364028dadc3a8bfda8d Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Tue, 23 Jun 2026 15:13:04 +0800 Subject: [PATCH] fix(redis): speed up large key scans --- .../src/components/redis/RedisKeyBrowser.vue | 84 ++++++++++++++----- .../src/components/redis/RedisValueViewer.vue | 6 +- apps/desktop/src/lib/http.ts | 4 +- apps/desktop/src/lib/tauri.ts | 4 +- apps/desktop/src/stores/connectionStore.ts | 2 +- crates/dbx-core/src/db/redis_driver.rs | 61 +++++++++++--- crates/dbx-core/src/redis_ops.rs | 29 +++++-- crates/dbx-web/src/routes/redis.rs | 2 + src-tauri/src/commands/redis_cmd.rs | 14 +++- 9 files changed, 154 insertions(+), 52 deletions(-) diff --git a/apps/desktop/src/components/redis/RedisKeyBrowser.vue b/apps/desktop/src/components/redis/RedisKeyBrowser.vue index 184eb342a..857d234b7 100644 --- a/apps/desktop/src/components/redis/RedisKeyBrowser.vue +++ b/apps/desktop/src/components/redis/RedisKeyBrowser.vue @@ -18,7 +18,7 @@ import RedisValueViewer from "./RedisValueViewer.vue"; import RedisPubSubPanel from "./RedisPubSubPanel.vue"; import RedisSlowlogPanel from "./RedisSlowlogPanel.vue"; import * as api from "@/lib/api"; -import type { RedisKeyInfo, RedisScanResult, HistoryEntry } from "@/lib/api"; +import type { RedisKeyInfo, RedisScanResult, RedisValue, HistoryEntry } from "@/lib/api"; import { uuid } from "@/lib/utils"; import { useConnectionStore } from "@/stores/connectionStore"; import { useSettingsStore } from "@/stores/settingsStore"; @@ -102,6 +102,7 @@ let nextEntryId = 0; let searchRequestId = 0; let redisBrowserIsActive = true; let redisDbFlushedListenerRegistered = false; +const loadedKeyRaws = new Set(); const valueQuery = computed(() => searchPattern.value.trim()); const isValueSearchMode = computed(() => searchMode.value === "value" || searchMode.value === "all"); @@ -200,24 +201,30 @@ function mergeTree(newKeys: RedisKeyInfo[]) { async function fetchScanPage(): Promise { const pageSize = settingsStore.editorSettings.redisScanPageSize; - return isValueSearchMode.value ? await api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all") : await api.redisScanKeys(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize); + return isValueSearchMode.value ? await api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all") : await api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, 1, false); } /// Batch-scan variant that performs multiple SCAN iterations server-side. /// Dramatically reduces frontend↔backend roundtrips for bulk loading. -async function fetchScanBatchPage(maxIterations: number): Promise { - const pageSize = settingsStore.editorSettings.redisScanPageSize; +async function fetchScanBatchPage(maxIterations: number, options: { count?: number; includeTypes?: boolean } = {}): Promise { + const pageSize = options.count ?? settingsStore.editorSettings.redisScanPageSize; // Value search cannot be batched because each key requires a GET. if (isValueSearchMode.value) { return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all"); } - return api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, maxIterations); + return api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, maxIterations, options.includeTypes ?? false); } -function appendScanResult(result: RedisScanResult) { - const existingKeys = new Set(flatKeys.value.map((key) => key.key_raw)); - const newKeys = result.keys.filter((key) => !existingKeys.has(key.key_raw)); - flatKeys.value = [...flatKeys.value, ...newKeys]; +function appendScanResult(result: RedisScanResult, options: { updateTree?: boolean } = {}) { + const newKeys: RedisKeyInfo[] = []; + for (const key of result.keys) { + if (loadedKeyRaws.has(key.key_raw)) continue; + loadedKeyRaws.add(key.key_raw); + newKeys.push(key); + } + if (newKeys.length > 0) { + flatKeys.value = [...flatKeys.value, ...newKeys]; + } scanCursor.value = result.cursor; hasMore.value = result.cursor !== 0; // DBSIZE is only called on the first batch page (cursor==0); subsequent @@ -228,10 +235,12 @@ function appendScanResult(result: RedisScanResult) { lastTotalKeys.value = result.total_keys; } - if (treeKeys.value.length === 0) { - rebuildTree(isSearchMode.value); - } else { - mergeTree(newKeys); + if (options.updateTree ?? true) { + if (treeKeys.value.length === 0) { + rebuildTree(isSearchMode.value); + } else { + mergeTree(newKeys); + } } connectionStore.updateRedisDbKeyStats(props.connectionId, props.db, { @@ -276,6 +285,7 @@ async function loadKeys() { const requestId = ++searchRequestId; isFetchingAll.value = false; loading.value = true; + loadedKeyRaws.clear(); flatKeys.value = []; treeKeys.value = []; selectedKeyRaw.value = null; @@ -313,25 +323,29 @@ async function loadMore() { } } -/// Fetch-all with server-side multi-SCAN batching. -/// -/// Each call performs up to 15 SCAN→TYPE cycles server-side (~0.5s per -/// batch at COUNT=1000). This keeps the UI responsive with frequent progress -/// updates while still avoiding the per-page overhead of single-SCAN calls. -const FETCH_ALL_BATCH_ITERATIONS = 15; +// Fetch-all uses large key-only SCAN pages and rebuilds the tree once at the +// end; per-page tree sorting dominates runtime on million-key pattern scans. +const FETCH_ALL_SCAN_COUNT = 50000; +const FETCH_ALL_BATCH_ITERATIONS = 1; async function fetchAll() { if (!hasMore.value || isFetchingAll.value) return; const requestId = searchRequestId; isFetchingAll.value = true; + let changed = false; try { while (requestId === searchRequestId && isFetchingAll.value && hasMore.value) { - const result = await fetchScanBatchPage(FETCH_ALL_BATCH_ITERATIONS); + const result = await fetchScanBatchPage(FETCH_ALL_BATCH_ITERATIONS, { + count: FETCH_ALL_SCAN_COUNT, + includeTypes: false, + }); if (requestId !== searchRequestId) break; - appendScanResult(result); + appendScanResult(result, { updateTree: false }); + changed = true; } } finally { if (requestId === searchRequestId) { + if (changed) rebuildTree(isSearchMode.value); isFetchingAll.value = false; } } @@ -360,6 +374,7 @@ function onRowClick(node: RedisKeyTreeNode) { function onKeyDeleted() { if (!selectedKeyRaw.value) return; + loadedKeyRaws.delete(selectedKeyRaw.value); flatKeys.value = flatKeys.value.filter((key) => key.key_raw !== selectedKeyRaw.value); selectedKeyRaw.value = null; rebuildTree(false); @@ -369,6 +384,26 @@ function onKeyDeleted() { }); } +function redisValueToKeyInfo(value: RedisValue): RedisKeyInfo { + return { + key_display: value.key_display, + key_raw: value.key_raw, + key_type: value.key_type, + ttl: value.ttl, + size: typeof value.value === "string" ? value.value.length : (value.total ?? 0), + value_preview: createdKeyPreview(value.value), + }; +} + +function onKeyLoaded(value: RedisValue) { + const keyInfo = redisValueToKeyInfo(value); + const existingIndex = flatKeys.value.findIndex((key) => key.key_raw === keyInfo.key_raw); + if (existingIndex < 0) return; + flatKeys.value = flatKeys.value.map((key, index) => (index === existingIndex ? keyInfo : key)); + loadedKeyRaws.add(keyInfo.key_raw); + rebuildTree(false); +} + function toggleCheck(keyRaw: string, event: Event) { event.stopPropagation(); const next = new Set(checkedKeys.value); @@ -393,6 +428,7 @@ function requestGroupDelete(node: RedisKeyTreeNode, event: Event) { } function resetLoadedKeys() { + loadedKeyRaws.clear(); flatKeys.value = []; treeKeys.value = []; selectedKeyRaw.value = null; @@ -404,6 +440,7 @@ function resetLoadedKeys() { async function deleteKeyRaws(keys: string[]) { const deletedCount = await api.redisDeleteKeys(props.connectionId, props.db, keys); const deleted = new Set(keys); + for (const key of deleted) loadedKeyRaws.delete(key); flatKeys.value = flatKeys.value.filter((k) => !deleted.has(k.key_raw)); if (selectedKeyRaw.value && deleted.has(selectedKeyRaw.value)) { selectedKeyRaw.value = null; @@ -601,6 +638,7 @@ function upsertCreatedKey(value: any) { } else { flatKeys.value = [keyInfo, ...flatKeys.value]; } + loadedKeyRaws.add(keyInfo.key_raw); selectedKeyRaw.value = keyInfo.key_raw; rebuildTree(isSearchMode.value); connectionStore.updateRedisDbKeyStats(props.connectionId, props.db, { @@ -974,7 +1012,7 @@ defineExpose({ focusSearch });
- {{ row.node.keyType }} + {{ row.node.keyType }} @@ -1031,7 +1069,7 @@ defineExpose({ focusSearch });
- +
{{ t("redis.selectKeyForDetail") }}
diff --git a/apps/desktop/src/components/redis/RedisValueViewer.vue b/apps/desktop/src/components/redis/RedisValueViewer.vue index 70b3e7a67..1e24e43d3 100644 --- a/apps/desktop/src/components/redis/RedisValueViewer.vue +++ b/apps/desktop/src/components/redis/RedisValueViewer.vue @@ -32,7 +32,7 @@ const props = defineProps<{ metadata?: RedisKeyInfo | null; }>(); -const emit = defineEmits<{ deleted: [] }>(); +const emit = defineEmits<{ deleted: []; loaded: [value: RedisValue] }>(); const data = ref(null); const loading = ref(false); @@ -199,7 +199,9 @@ async function load(options: { selectDefaultMember?: boolean } = {}) { const shouldSelectDefaultMember = options.selectDefaultMember ?? true; loading.value = true; try { - data.value = await api.redisGetValue(props.connectionId, props.db, props.keyRaw); + const loadedValue = await api.redisGetValue(props.connectionId, props.db, props.keyRaw); + data.value = loadedValue; + emit("loaded", loadedValue); scanCursor.value = data.value.scan_cursor ?? undefined; if (data.value.key_type === "string") { const detail = formatRedisMemberDetail(data.value.value); diff --git a/apps/desktop/src/lib/http.ts b/apps/desktop/src/lib/http.ts index e51b60e55..91a781a60 100644 --- a/apps/desktop/src/lib/http.ts +++ b/apps/desktop/src/lib/http.ts @@ -1366,8 +1366,8 @@ export async function redisScanKeys(connectionId: string, db: number, cursor: nu return post("/api/redis/scan-keys", { connectionId, db, cursor, pattern, count }); } -export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number): Promise { - return post("/api/redis/scan-keys-batch", { connectionId, db, cursor, pattern, count, maxIterations }); +export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number, includeTypes = true): Promise { + return post("/api/redis/scan-keys-batch", { connectionId, db, cursor, pattern, count, maxIterations, includeTypes }); } export async function redisScanValues(connectionId: string, db: number, cursor: number, pattern: string, query: string, count: number, includeKeyMatches = false): Promise { diff --git a/apps/desktop/src/lib/tauri.ts b/apps/desktop/src/lib/tauri.ts index 68fe2940c..97d671994 100644 --- a/apps/desktop/src/lib/tauri.ts +++ b/apps/desktop/src/lib/tauri.ts @@ -1163,8 +1163,8 @@ export async function redisScanKeys(connectionId: string, db: number, cursor: nu return invoke("redis_scan_keys", { connectionId, db, cursor, pattern, count }); } -export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number): Promise { - return invoke("redis_scan_keys_batch", { connectionId, db, cursor, pattern, count, maxIterations }); +export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number, includeTypes = true): Promise { + return invoke("redis_scan_keys_batch", { connectionId, db, cursor, pattern, count, maxIterations, includeTypes }); } export async function redisScanValues(connectionId: string, db: number, cursor: number, pattern: string, query: string, count: number, includeKeyMatches = false): Promise { diff --git a/apps/desktop/src/stores/connectionStore.ts b/apps/desktop/src/stores/connectionStore.ts index 2dd1242a6..6809b990b 100644 --- a/apps/desktop/src/stores/connectionStore.ts +++ b/apps/desktop/src/stores/connectionStore.ts @@ -2358,7 +2358,7 @@ export const useConnectionStore = defineStore("connection", () => { await ensureConnected(connectionId); const pageSize = settingsStore.editorSettings.redisScanPageSize; // Bounded multi-round SCAN: trade coverage for latency/memory safety. - const result = await api.redisScanKeysBatch(connectionId, Number(database), 0, "*", pageSize, 6); + const result = await api.redisScanKeysBatch(connectionId, Number(database), 0, "*", pageSize, 6, false); const keys = result.keys.map((key) => key.key_display).slice(0, REDIS_COMPLETION_KEYS_MAX); redisCompletionKeysCache.value[cacheKey] = keys; evictOldestCacheEntries(redisCompletionKeysCache.value, COMPLETION_CACHE_MAX); diff --git a/crates/dbx-core/src/db/redis_driver.rs b/crates/dbx-core/src/db/redis_driver.rs index beb5dc1cb..68674ee44 100644 --- a/crates/dbx-core/src/db/redis_driver.rs +++ b/crates/dbx-core/src/db/redis_driver.rs @@ -760,6 +760,16 @@ pub async fn scan_cluster_keys_page( cursor: u64, pattern: &str, count: usize, +) -> Result { + scan_cluster_keys_page_with_options(pool, cursor, pattern, count, true).await +} + +pub async fn scan_cluster_keys_page_with_options( + pool: &RedisClusterPool, + cursor: u64, + pattern: &str, + count: usize, + include_types: bool, ) -> Result { let master_nodes = cluster_master_nodes(pool).await?; if master_nodes.is_empty() { @@ -776,7 +786,7 @@ pub async fn scan_cluster_keys_page( let endpoint = &master_nodes[index]; let mut con = connect_cluster_node(pool, endpoint).await?; let current_cursor = if index == node_index { node_cursor } else { 0 }; - let result = scan_keys_page(&mut con, current_cursor, pattern, count).await?; + let result = scan_keys_page_with_options(&mut con, current_cursor, pattern, count, include_types).await?; if !result.keys.is_empty() { let next_cursor = if result.cursor != 0 { encode_cluster_cursor(index, result.cursor)? @@ -1434,13 +1444,26 @@ pub async fn scan_keys_page(con: &mut C, cursor: u64, pattern: &str, count: u where C: ConnectionLike + Send + Sync + Unpin, { - scan_keys_batch(con, cursor, pattern, count, 1).await + scan_keys_batch(con, cursor, pattern, count, 1, true).await +} + +pub async fn scan_keys_page_with_options( + con: &mut C, + cursor: u64, + pattern: &str, + count: usize, + include_types: bool, +) -> Result +where + C: ConnectionLike + Send + Sync + Unpin, +{ + scan_keys_batch(con, cursor, pattern, count, 1, include_types).await } /// Batch-scan keys with server-side multi-SCAN support. /// -/// Performs up to `max_iterations` SCAN→TYPE cycles in a single call, -/// dramatically reducing frontend↔backend roundtrips when fetching many keys. +/// Performs up to `max_iterations` SCAN cycles in a single call. TYPE metadata +/// is optional so large key-name searches can avoid extra Redis work. /// DBSIZE is only called on the first iteration (cursor == 0). pub async fn scan_keys_batch( con: &mut C, @@ -1448,6 +1471,7 @@ pub async fn scan_keys_batch( pattern: &str, count: usize, max_iterations: usize, + include_types: bool, ) -> Result where C: ConnectionLike + Send + Sync + Unpin, @@ -1472,23 +1496,34 @@ where let (next_cursor, keys) = parse_scan_keys(raw)?; if !keys.is_empty() { - let mut pipe = redis::pipe(); - for key in &keys { - pipe.cmd("TYPE").arg(key); - } - let key_types: Vec = pipe.query_async(con).await.unwrap_or_default(); + let key_types: Vec = if include_types { + let mut pipe = redis::pipe(); + for key in &keys { + pipe.cmd("TYPE").arg(key); + } + pipe.query_async(con).await.unwrap_or_default() + } else { + Vec::new() + }; for (index, key) in keys.iter().enumerate() { - let key_type = key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string()); + let key_type = if include_types { + key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string()) + } else { + "unknown".to_string() + }; + let value_preview = if include_types { + redis_key_value_preview(key_types.get(index).map(String::as_str).unwrap_or("unknown")) + } else { + String::new() + }; all_keys.push(RedisKeyInfo { key_display: redis_key_bytes_to_display(key), key_raw: redis_key_bytes_to_raw(key), key_type, ttl: -2, size: 0, - value_preview: redis_key_value_preview( - key_types.get(index).map(String::as_str).unwrap_or("unknown"), - ), + value_preview, }); } } diff --git a/crates/dbx-core/src/redis_ops.rs b/crates/dbx-core/src/redis_ops.rs index 651d60e1e..21e2986ec 100644 --- a/crates/dbx-core/src/redis_ops.rs +++ b/crates/dbx-core/src/redis_ops.rs @@ -34,14 +34,14 @@ pub async fn redis_scan_keys_core( pattern: &str, count: usize, ) -> Result { - redis_scan_keys_batch_core(state, connection_id, db, cursor, pattern, count, 1).await + redis_scan_keys_batch_core(state, connection_id, db, cursor, pattern, count, 1, true).await } /// Batch-scan keys with server-side multi-SCAN support. /// -/// Performs up to `max_iterations` SCAN→TYPE cycles server-side in a single -/// API call, dramatically reducing frontend↔backend roundtrips when fetching -/// many keys (e.g. "fetch all" in the key browser). +/// Performs up to `max_iterations` SCAN cycles server-side in a single API +/// call, dramatically reducing frontend↔backend roundtrips when fetching many +/// keys (e.g. "fetch all" in the key browser). TYPE metadata is optional. pub async fn redis_scan_keys_batch_core( state: &AppState, connection_id: &str, @@ -50,6 +50,7 @@ pub async fn redis_scan_keys_batch_core( pattern: &str, count: usize, max_iterations: usize, + include_types: bool, ) -> Result { ensure_redis_pool(state, connection_id).await?; let connections = state.connections.read().await; @@ -59,20 +60,34 @@ pub async fn redis_scan_keys_batch_core( RedisConnection::Direct(con) => { let mut con = con.lock().await; redis_driver::select_db(&mut *con, db).await?; - redis_driver::scan_keys_batch(&mut *con, cursor, pattern, count, max_iterations).await + redis_driver::scan_keys_batch(&mut *con, cursor, pattern, count, max_iterations, include_types).await } RedisConnection::Cluster(cluster) => { redis_driver::ensure_cluster_db(db)?; // Cluster scan already iterates across nodes; for batch mode we // loop the cluster-level scan to accumulate keys server-side. if max_iterations <= 1 { - return redis_driver::scan_cluster_keys_page(cluster, cursor, pattern, count).await; + return redis_driver::scan_cluster_keys_page_with_options( + cluster, + cursor, + pattern, + count, + include_types, + ) + .await; } let mut all_keys: Vec = Vec::new(); let mut current_cursor = cursor; let mut total_keys: u64 = 0; for i in 0..max_iterations { - let page = redis_driver::scan_cluster_keys_page(cluster, current_cursor, pattern, count).await?; + let page = redis_driver::scan_cluster_keys_page_with_options( + cluster, + current_cursor, + pattern, + count, + include_types, + ) + .await?; if i == 0 { total_keys = page.total_keys; } diff --git a/crates/dbx-web/src/routes/redis.rs b/crates/dbx-web/src/routes/redis.rs index 5959190fd..eceb090fa 100644 --- a/crates/dbx-web/src/routes/redis.rs +++ b/crates/dbx-web/src/routes/redis.rs @@ -48,6 +48,7 @@ pub struct RedisScanBatchRequest { pub count: usize, #[serde(default = "default_max_iterations")] pub max_iterations: usize, + pub include_types: Option, } fn default_max_iterations() -> usize { @@ -234,6 +235,7 @@ pub async fn scan_keys_batch( &req.pattern, req.count, req.max_iterations, + req.include_types.unwrap_or(true), ) .await .map_err(AppError)?; diff --git a/src-tauri/src/commands/redis_cmd.rs b/src-tauri/src/commands/redis_cmd.rs index 6deebe6c1..e0637e0f8 100644 --- a/src-tauri/src/commands/redis_cmd.rs +++ b/src-tauri/src/commands/redis_cmd.rs @@ -35,9 +35,19 @@ pub async fn redis_scan_keys_batch( pattern: String, count: usize, max_iterations: usize, + include_types: Option, ) -> Result { - dbx_core::redis_ops::redis_scan_keys_batch_core(&state, &connection_id, db, cursor, &pattern, count, max_iterations) - .await + dbx_core::redis_ops::redis_scan_keys_batch_core( + &state, + &connection_id, + db, + cursor, + &pattern, + count, + max_iterations, + include_types.unwrap_or(true), + ) + .await } #[tauri::command]