From ab5bd1dd252928f228a55400b08ca162eb5c94e2 Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Mon, 15 Jun 2026 15:01:13 +0800 Subject: [PATCH] fix(redis): batch SCAN server-side to reduce fetch-all latency --- .../src/components/redis/RedisKeyBrowser.vue | 54 +++++++--- apps/desktop/src/lib/api.ts | 1 + apps/desktop/src/lib/http.ts | 4 + apps/desktop/src/lib/tauri.ts | 4 + crates/dbx-core/src/db/redis_driver.rs | 98 ++++++++++++------- crates/dbx-core/src/redis_ops.rs | 42 +++++++- crates/dbx-web/src/main.rs | 1 + crates/dbx-web/src/routes/redis.rs | 34 +++++++ src-tauri/src/commands/redis_cmd.rs | 14 +++ src-tauri/src/lib.rs | 1 + 10 files changed, 204 insertions(+), 49 deletions(-) diff --git a/apps/desktop/src/components/redis/RedisKeyBrowser.vue b/apps/desktop/src/components/redis/RedisKeyBrowser.vue index 6301867b0..dcfdc3977 100644 --- a/apps/desktop/src/components/redis/RedisKeyBrowser.vue +++ b/apps/desktop/src/components/redis/RedisKeyBrowser.vue @@ -200,13 +200,30 @@ async function fetchScanPage(): Promise { return isValueSearchMode.value ? await api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all") : await api.redisScanKeys(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize); } +/// Batch-scan variant that performs multiple SCAN iterations server-side. +/// Dramatically reduces frontend↔backend roundtrips for bulk loading. +async function fetchScanBatchPage(maxIterations: number): Promise { + const pageSize = settingsStore.editorSettings.redisScanPageSize; + // Value search cannot be batched because each key requires a GET. + if (isValueSearchMode.value) { + return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all"); + } + return api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, maxIterations); +} + function appendScanResult(result: RedisScanResult) { const existingKeys = new Set(flatKeys.value.map((key) => key.key_raw)); const newKeys = result.keys.filter((key) => !existingKeys.has(key.key_raw)); flatKeys.value = [...flatKeys.value, ...newKeys]; scanCursor.value = result.cursor; hasMore.value = result.cursor !== 0; - lastTotalKeys.value = result.total_keys; + // DBSIZE is only called on the first batch page (cursor==0); subsequent + // pages return total_keys=0. Preserve the previously-fetched total when + // we get a zero from a continuation. A truly empty DB returns cursor==0 + // and keys==[] along with total_keys==0, which we do record. + if (result.total_keys > 0 || (result.cursor === 0 && result.keys.length === 0)) { + lastTotalKeys.value = result.total_keys; + } if (treeKeys.value.length === 0) { rebuildTree(isSearchMode.value); @@ -216,7 +233,7 @@ function appendScanResult(result: RedisScanResult) { connectionStore.updateRedisDbKeyStats(props.connectionId, props.db, { loaded: isSearchMode.value ? undefined : flatKeys.value.length, - total: result.total_keys, + total: result.total_keys > 0 || (result.cursor === 0 && result.keys.length === 0) ? result.total_keys : undefined, }); } @@ -236,15 +253,18 @@ async function streamValueSearch(requestId: number) { async function fillInitialKeyBatch(requestId: number) { const targetCount = Math.max(1, settingsStore.editorSettings.redisScanPageSize); - let rounds = 0; - while (requestId === searchRequestId && searchMode.value === "key" && hasMore.value && flatKeys.value.length < targetCount) { - const beforeCount = flatKeys.value.length; - const applied = await scanNextPage(requestId); - if (!applied) return; - rounds += 1; - if (flatKeys.value.length >= targetCount) return; - if (rounds >= 12 && flatKeys.value.length === beforeCount) return; - if (rounds >= 24) return; + // Use server-side batching to fill the initial view quickly. + // Each batch iteration does one SCAN round-trip; 5 iterations with + // COUNT=1000 should return enough keys for most cases. + const maxIter = Math.max(1, Math.ceil(targetCount / Math.max(1, settingsStore.editorSettings.redisScanPageSize))); + const result = await fetchScanBatchPage(Math.min(maxIter, 8)); + if (requestId !== searchRequestId) return; + appendScanResult(result); + // If we still need more keys, do one more batch + if (flatKeys.value.length < targetCount && hasMore.value && requestId === searchRequestId) { + const result2 = await fetchScanBatchPage(Math.min(maxIter, 8)); + if (requestId !== searchRequestId) return; + appendScanResult(result2); } } @@ -290,14 +310,22 @@ async function loadMore() { } } +/// Fetch-all with server-side multi-SCAN batching. +/// +/// Each call performs up to 15 SCAN→TYPE cycles server-side (~0.5s per +/// batch at COUNT=1000). This keeps the UI responsive with frequent progress +/// updates while still avoiding the per-page overhead of single-SCAN calls. +const FETCH_ALL_BATCH_ITERATIONS = 15; + async function fetchAll() { if (!hasMore.value || isFetchingAll.value) return; const requestId = searchRequestId; isFetchingAll.value = true; try { while (requestId === searchRequestId && isFetchingAll.value && hasMore.value) { - const applied = await scanNextPage(requestId); - if (!applied) break; + const result = await fetchScanBatchPage(FETCH_ALL_BATCH_ITERATIONS); + if (requestId !== searchRequestId) break; + appendScanResult(result); } } finally { if (requestId === searchRequestId) { diff --git a/apps/desktop/src/lib/api.ts b/apps/desktop/src/lib/api.ts index 58122d7c0..bfcb15d64 100644 --- a/apps/desktop/src/lib/api.ts +++ b/apps/desktop/src/lib/api.ts @@ -240,6 +240,7 @@ export const cancelTableExport = forward("cancelTableExport"); // Redis export const redisListDatabases = forward("redisListDatabases"); export const redisScanKeys = forward("redisScanKeys"); +export const redisScanKeysBatch = forward("redisScanKeysBatch"); export const redisScanValues = forward("redisScanValues"); export const redisGetValue = forward("redisGetValue"); export const redisSetString = forward("redisSetString"); diff --git a/apps/desktop/src/lib/http.ts b/apps/desktop/src/lib/http.ts index bd1b245d9..d4367fe7c 100644 --- a/apps/desktop/src/lib/http.ts +++ b/apps/desktop/src/lib/http.ts @@ -1270,6 +1270,10 @@ export async function redisScanKeys(connectionId: string, db: number, cursor: nu return post("/api/redis/scan-keys", { connectionId, db, cursor, pattern, count }); } +export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number): Promise { + return post("/api/redis/scan-keys-batch", { connectionId, db, cursor, pattern, count, maxIterations }); +} + export async function redisScanValues(connectionId: string, db: number, cursor: number, pattern: string, query: string, count: number, includeKeyMatches = false): Promise { return post("/api/redis/scan-values", { connectionId, db, cursor, pattern, query, includeKeyMatches, count }); } diff --git a/apps/desktop/src/lib/tauri.ts b/apps/desktop/src/lib/tauri.ts index a2297955b..4e9ef1c4d 100644 --- a/apps/desktop/src/lib/tauri.ts +++ b/apps/desktop/src/lib/tauri.ts @@ -1108,6 +1108,10 @@ export async function redisScanKeys(connectionId: string, db: number, cursor: nu return invoke("redis_scan_keys", { connectionId, db, cursor, pattern, count }); } +export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number): Promise { + return invoke("redis_scan_keys_batch", { connectionId, db, cursor, pattern, count, maxIterations }); +} + export async function redisScanValues(connectionId: string, db: number, cursor: number, pattern: string, query: string, count: number, includeKeyMatches = false): Promise { return invoke("redis_scan_values", { connectionId, db, cursor, pattern, query, includeKeyMatches, count }); } diff --git a/crates/dbx-core/src/db/redis_driver.rs b/crates/dbx-core/src/db/redis_driver.rs index 19e767712..8f8bef8ec 100644 --- a/crates/dbx-core/src/db/redis_driver.rs +++ b/crates/dbx-core/src/db/redis_driver.rs @@ -1000,41 +1000,72 @@ pub async fn scan_keys_page(con: &mut C, cursor: u64, pattern: &str, count: u where C: ConnectionLike + Send + Sync + Unpin, { - let raw: RedisRawValue = redis::cmd("SCAN") - .arg(cursor) - .arg("MATCH") - .arg(pattern) - .arg("COUNT") - .arg(count) - .query_async(con) - .await - .map_err(|e| e.to_string())?; + scan_keys_batch(con, cursor, pattern, count, 1).await +} - let (next_cursor, keys) = parse_scan_keys(raw)?; - let total_keys: u64 = redis::cmd("DBSIZE").query_async(con).await.unwrap_or(0); - if keys.is_empty() { - return Ok(RedisScanResult { cursor: next_cursor, keys: Vec::new(), total_keys }); +/// Batch-scan keys with server-side multi-SCAN support. +/// +/// Performs up to `max_iterations` SCAN→TYPE cycles in a single call, +/// dramatically reducing frontend↔backend roundtrips when fetching many keys. +/// DBSIZE is only called on the first iteration (cursor == 0). +pub async fn scan_keys_batch( + con: &mut C, + cursor: u64, + pattern: &str, + count: usize, + max_iterations: usize, +) -> Result +where + C: ConnectionLike + Send + Sync + Unpin, +{ + let iterations = max_iterations.max(1); + let total_keys: u64 = if cursor == 0 { redis::cmd("DBSIZE").query_async(con).await.unwrap_or(0) } else { 0 }; + + let mut all_keys: Vec = Vec::new(); + let mut current_cursor = cursor; + + for _ in 0..iterations { + let raw: RedisRawValue = redis::cmd("SCAN") + .arg(current_cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(count) + .query_async(con) + .await + .map_err(|e| e.to_string())?; + + let (next_cursor, keys) = parse_scan_keys(raw)?; + + if !keys.is_empty() { + let mut pipe = redis::pipe(); + for key in &keys { + pipe.cmd("TYPE").arg(key); + } + let key_types: Vec = pipe.query_async(con).await.unwrap_or_default(); + + for (index, key) in keys.iter().enumerate() { + let key_type = key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string()); + all_keys.push(RedisKeyInfo { + key_display: redis_key_bytes_to_display(key), + key_raw: redis_key_bytes_to_raw(key), + key_type, + ttl: -2, + size: 0, + value_preview: redis_key_value_preview( + key_types.get(index).map(String::as_str).unwrap_or("unknown"), + ), + }); + } + } + + if next_cursor == 0 { + return Ok(RedisScanResult { cursor: 0, keys: all_keys, total_keys }); + } + current_cursor = next_cursor; } - let mut pipe = redis::pipe(); - for key in &keys { - pipe.cmd("TYPE").arg(key); - } - let key_types: Vec = pipe.query_async(con).await.unwrap_or_default(); - - let mut result = Vec::with_capacity(keys.len()); - for (index, key) in keys.iter().enumerate() { - let key_type = key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string()); - result.push(RedisKeyInfo { - key_display: redis_key_bytes_to_display(key), - key_raw: redis_key_bytes_to_raw(key), - key_type, - ttl: -2, - size: 0, - value_preview: redis_key_value_preview(key_types.get(index).map(String::as_str).unwrap_or("unknown")), - }); - } - Ok(RedisScanResult { cursor: next_cursor, keys: result, total_keys }) + Ok(RedisScanResult { cursor: current_cursor, keys: all_keys, total_keys }) } pub async fn scan_values_page( @@ -1048,7 +1079,8 @@ pub async fn scan_values_page( where C: ConnectionLike + Send + Sync + Unpin, { - let total_keys: u64 = redis::cmd("DBSIZE").query_async(con).await.unwrap_or(0); + // Only call DBSIZE on the first page (cursor == 0) to avoid redundant work. + let total_keys: u64 = if cursor == 0 { redis::cmd("DBSIZE").query_async(con).await.unwrap_or(0) } else { 0 }; if query.trim().is_empty() { return Ok(RedisScanResult { cursor, keys: Vec::new(), total_keys }); } diff --git a/crates/dbx-core/src/redis_ops.rs b/crates/dbx-core/src/redis_ops.rs index 7a0a577ab..c0928c721 100644 --- a/crates/dbx-core/src/redis_ops.rs +++ b/crates/dbx-core/src/redis_ops.rs @@ -1,6 +1,6 @@ use crate::connection::{AppState, PoolKind}; use crate::db::redis_driver::{ - self, RedisCommandResult, RedisConnection, RedisDatabaseInfo, RedisScanResult, RedisValue, + self, RedisCommandResult, RedisConnection, RedisDatabaseInfo, RedisKeyInfo, RedisScanResult, RedisValue, }; async fn ensure_redis_pool(state: &AppState, connection_id: &str) -> Result<(), String> { @@ -33,6 +33,23 @@ pub async fn redis_scan_keys_core( cursor: u64, pattern: &str, count: usize, +) -> Result { + redis_scan_keys_batch_core(state, connection_id, db, cursor, pattern, count, 1).await +} + +/// Batch-scan keys with server-side multi-SCAN support. +/// +/// Performs up to `max_iterations` SCAN→TYPE cycles server-side in a single +/// API call, dramatically reducing frontend↔backend roundtrips when fetching +/// many keys (e.g. "fetch all" in the key browser). +pub async fn redis_scan_keys_batch_core( + state: &AppState, + connection_id: &str, + db: u32, + cursor: u64, + pattern: &str, + count: usize, + max_iterations: usize, ) -> Result { ensure_redis_pool(state, connection_id).await?; let connections = state.connections.read().await; @@ -42,11 +59,30 @@ pub async fn redis_scan_keys_core( RedisConnection::Direct(con) => { let mut con = con.lock().await; redis_driver::select_db(&mut *con, db).await?; - redis_driver::scan_keys_page(&mut *con, cursor, pattern, count).await + redis_driver::scan_keys_batch(&mut *con, cursor, pattern, count, max_iterations).await } RedisConnection::Cluster(cluster) => { redis_driver::ensure_cluster_db(db)?; - redis_driver::scan_cluster_keys_page(cluster, cursor, pattern, count).await + // Cluster scan already iterates across nodes; for batch mode we + // loop the cluster-level scan to accumulate keys server-side. + if max_iterations <= 1 { + return redis_driver::scan_cluster_keys_page(cluster, cursor, pattern, count).await; + } + let mut all_keys: Vec = Vec::new(); + let mut current_cursor = cursor; + let mut total_keys: u64 = 0; + for i in 0..max_iterations { + let page = redis_driver::scan_cluster_keys_page(cluster, current_cursor, pattern, count).await?; + if i == 0 { + total_keys = page.total_keys; + } + all_keys.extend(page.keys); + if page.cursor == 0 { + return Ok(RedisScanResult { cursor: 0, keys: all_keys, total_keys }); + } + current_cursor = page.cursor; + } + Ok(RedisScanResult { cursor: current_cursor, keys: all_keys, total_keys }) } }, _ => Err("Not a Redis connection".to_string()), diff --git a/crates/dbx-web/src/main.rs b/crates/dbx-web/src/main.rs index 82c1f748e..9fc2dbba9 100644 --- a/crates/dbx-web/src/main.rs +++ b/crates/dbx-web/src/main.rs @@ -240,6 +240,7 @@ async fn main() { // Redis .route("/redis/list-databases", post(routes::redis::list_databases)) .route("/redis/scan-keys", post(routes::redis::scan_keys)) + .route("/redis/scan-keys-batch", post(routes::redis::scan_keys_batch)) .route("/redis/scan-values", post(routes::redis::scan_values)) .route("/redis/get-value", post(routes::redis::get_value)) .route("/redis/set-string", post(routes::redis::set_string)) diff --git a/crates/dbx-web/src/routes/redis.rs b/crates/dbx-web/src/routes/redis.rs index 30af50818..e1d51bc4a 100644 --- a/crates/dbx-web/src/routes/redis.rs +++ b/crates/dbx-web/src/routes/redis.rs @@ -38,6 +38,22 @@ pub struct RedisScanRequest { pub count: usize, } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RedisScanBatchRequest { + pub connection_id: String, + pub db: u32, + pub cursor: u64, + pub pattern: String, + pub count: usize, + #[serde(default = "default_max_iterations")] + pub max_iterations: usize, +} + +fn default_max_iterations() -> usize { + 1 +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct RedisValueScanRequest { @@ -191,6 +207,24 @@ pub async fn scan_keys( Ok(Json(serde_json::to_value(result).map_err(|e| AppError(e.to_string()))?)) } +pub async fn scan_keys_batch( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + let result = dbx_core::redis_ops::redis_scan_keys_batch_core( + &state.app, + &req.connection_id, + req.db, + req.cursor, + &req.pattern, + req.count, + req.max_iterations, + ) + .await + .map_err(AppError)?; + Ok(Json(serde_json::to_value(result).map_err(|e| AppError(e.to_string()))?)) +} + pub async fn scan_values( State(state): State>, Json(req): Json, diff --git a/src-tauri/src/commands/redis_cmd.rs b/src-tauri/src/commands/redis_cmd.rs index 4bfbdf843..819cdc548 100644 --- a/src-tauri/src/commands/redis_cmd.rs +++ b/src-tauri/src/commands/redis_cmd.rs @@ -26,6 +26,20 @@ pub async fn redis_scan_keys( dbx_core::redis_ops::redis_scan_keys_core(&state, &connection_id, db, cursor, &pattern, count).await } +#[tauri::command] +pub async fn redis_scan_keys_batch( + state: State<'_, Arc>, + connection_id: String, + db: u32, + cursor: u64, + pattern: String, + count: usize, + max_iterations: usize, +) -> Result { + dbx_core::redis_ops::redis_scan_keys_batch_core(&state, &connection_id, db, cursor, &pattern, count, max_iterations) + .await +} + #[tauri::command] pub async fn redis_scan_values( state: State<'_, Arc>, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 9aae747ab..8a388ae00 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -464,6 +464,7 @@ pub fn run() { commands::table_import::cancel_table_import, commands::redis_cmd::redis_list_databases, commands::redis_cmd::redis_scan_keys, + commands::redis_cmd::redis_scan_keys_batch, commands::redis_cmd::redis_scan_values, commands::redis_cmd::redis_get_value, commands::redis_cmd::redis_set_string,