xiaoxue_admin
/
dbx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(redis): batch SCAN server-side to reduce fetch-all latency

This commit is contained in:
t8y2 2026-06-15 15:01:13 +08:00
parent 08fb5bbf4d
commit ab5bd1dd25
10 changed files with 204 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
apps/desktop/src/components/redis/RedisKeyBrowser.vue
View File

@ -200,13 +200,30 @@ async function fetchScanPage(): Promise<RedisScanResult> {
return isValueSearchMode.value ? await api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all") : await api.redisScanKeys(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize);
}
/// Batch-scan variant that performs multiple SCAN iterations server-side.
/// Dramatically reduces frontend↔backend roundtrips for bulk loading.
async function fetchScanBatchPage(maxIterations: number): Promise<RedisScanResult> {
const pageSize = settingsStore.editorSettings.redisScanPageSize;
// Value search cannot be batched because each key requires a GET.
if (isValueSearchMode.value) {
return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all");
}
return api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, maxIterations);
}
function appendScanResult(result: RedisScanResult) {
const existingKeys = new Set(flatKeys.value.map((key) => key.key_raw));
const newKeys = result.keys.filter((key) => !existingKeys.has(key.key_raw));
flatKeys.value = [...flatKeys.value, ...newKeys];
scanCursor.value = result.cursor;
hasMore.value = result.cursor !== 0;
lastTotalKeys.value = result.total_keys;
// DBSIZE is only called on the first batch page (cursor==0); subsequent
// pages return total_keys=0. Preserve the previously-fetched total when
// we get a zero from a continuation. A truly empty DB returns cursor==0
// and keys==[] along with total_keys==0, which we do record.
if (result.total_keys > 0 || (result.cursor === 0 && result.keys.length === 0)) {
lastTotalKeys.value = result.total_keys;
}
if (treeKeys.value.length === 0) {
rebuildTree(isSearchMode.value);
@ -216,7 +233,7 @@ function appendScanResult(result: RedisScanResult) {
connectionStore.updateRedisDbKeyStats(props.connectionId, props.db, {
loaded: isSearchMode.value ? undefined : flatKeys.value.length,
total: result.total_keys,
total: result.total_keys > 0 || (result.cursor === 0 && result.keys.length === 0) ? result.total_keys : undefined,
});
}
@ -236,15 +253,18 @@ async function streamValueSearch(requestId: number) {
async function fillInitialKeyBatch(requestId: number) {
const targetCount = Math.max(1, settingsStore.editorSettings.redisScanPageSize);
let rounds = 0;
while (requestId === searchRequestId && searchMode.value === "key" && hasMore.value && flatKeys.value.length < targetCount) {
const beforeCount = flatKeys.value.length;
const applied = await scanNextPage(requestId);
if (!applied) return;
rounds += 1;
if (flatKeys.value.length >= targetCount) return;
if (rounds >= 12 && flatKeys.value.length === beforeCount) return;
if (rounds >= 24) return;
// Use server-side batching to fill the initial view quickly.
// Each batch iteration does one SCAN round-trip; 5 iterations with
// COUNT=1000 should return enough keys for most cases.
const maxIter = Math.max(1, Math.ceil(targetCount / Math.max(1, settingsStore.editorSettings.redisScanPageSize)));
const result = await fetchScanBatchPage(Math.min(maxIter, 8));
if (requestId !== searchRequestId) return;
appendScanResult(result);
// If we still need more keys, do one more batch
if (flatKeys.value.length < targetCount && hasMore.value && requestId === searchRequestId) {
const result2 = await fetchScanBatchPage(Math.min(maxIter, 8));
if (requestId !== searchRequestId) return;
appendScanResult(result2);
}
}
@ -290,14 +310,22 @@ async function loadMore() {
}
}
/// Fetch-all with server-side multi-SCAN batching.
///
/// Each call performs up to 15 SCAN→TYPE cycles server-side (~0.5s per
/// batch at COUNT=1000). This keeps the UI responsive with frequent progress
/// updates while still avoiding the per-page overhead of single-SCAN calls.
const FETCH_ALL_BATCH_ITERATIONS = 15;
async function fetchAll() {
if (!hasMore.value || isFetchingAll.value) return;
const requestId = searchRequestId;
isFetchingAll.value = true;
try {
while (requestId === searchRequestId && isFetchingAll.value && hasMore.value) {
const applied = await scanNextPage(requestId);
if (!applied) break;
const result = await fetchScanBatchPage(FETCH_ALL_BATCH_ITERATIONS);
if (requestId !== searchRequestId) break;
appendScanResult(result);
}
} finally {
if (requestId === searchRequestId) {

1
apps/desktop/src/lib/api.ts
View File

@ -240,6 +240,7 @@ export const cancelTableExport = forward("cancelTableExport");
// Redis
export const redisListDatabases = forward("redisListDatabases");
export const redisScanKeys = forward("redisScanKeys");
export const redisScanKeysBatch = forward("redisScanKeysBatch");
export const redisScanValues = forward("redisScanValues");
export const redisGetValue = forward("redisGetValue");
export const redisSetString = forward("redisSetString");

4
apps/desktop/src/lib/http.ts
View File

@ -1270,6 +1270,10 @@ export async function redisScanKeys(connectionId: string, db: number, cursor: nu
return post("/api/redis/scan-keys", { connectionId, db, cursor, pattern, count });
}
export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number): Promise<RedisScanResult> {
return post("/api/redis/scan-keys-batch", { connectionId, db, cursor, pattern, count, maxIterations });
}
export async function redisScanValues(connectionId: string, db: number, cursor: number, pattern: string, query: string, count: number, includeKeyMatches = false): Promise<RedisScanResult> {
return post("/api/redis/scan-values", { connectionId, db, cursor, pattern, query, includeKeyMatches, count });
}

4
apps/desktop/src/lib/tauri.ts
View File

@ -1108,6 +1108,10 @@ export async function redisScanKeys(connectionId: string, db: number, cursor: nu
return invoke("redis_scan_keys", { connectionId, db, cursor, pattern, count });
}
export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number): Promise<RedisScanResult> {
return invoke("redis_scan_keys_batch", { connectionId, db, cursor, pattern, count, maxIterations });
}
export async function redisScanValues(connectionId: string, db: number, cursor: number, pattern: string, query: string, count: number, includeKeyMatches = false): Promise<RedisScanResult> {
return invoke("redis_scan_values", { connectionId, db, cursor, pattern, query, includeKeyMatches, count });
}

98
crates/dbx-core/src/db/redis_driver.rs
View File

@ -1000,41 +1000,72 @@ pub async fn scan_keys_page<C>(con: &mut C, cursor: u64, pattern: &str, count: u
where
C: ConnectionLike + Send + Sync + Unpin,
{
let raw: RedisRawValue = redis::cmd("SCAN")
.arg(cursor)
.arg("MATCH")
.arg(pattern)
.arg("COUNT")
.arg(count)
.query_async(con)
.await
.map_err(|e| e.to_string())?;
scan_keys_batch(con, cursor, pattern, count, 1).await
}
let (next_cursor, keys) = parse_scan_keys(raw)?;
let total_keys: u64 = redis::cmd("DBSIZE").query_async(con).await.unwrap_or(0);
if keys.is_empty() {
return Ok(RedisScanResult { cursor: next_cursor, keys: Vec::new(), total_keys });
/// Batch-scan keys with server-side multi-SCAN support.
///
/// Performs up to `max_iterations` SCAN→TYPE cycles in a single call,
/// dramatically reducing frontend↔backend roundtrips when fetching many keys.
/// DBSIZE is only called on the first iteration (cursor == 0).
pub async fn scan_keys_batch<C>(
con: &mut C,
cursor: u64,
pattern: &str,
count: usize,
max_iterations: usize,
) -> Result<RedisScanResult, String>
where
C: ConnectionLike + Send + Sync + Unpin,
{
let iterations = max_iterations.max(1);
let total_keys: u64 = if cursor == 0 { redis::cmd("DBSIZE").query_async(con).await.unwrap_or(0) } else { 0 };
let mut all_keys: Vec<RedisKeyInfo> = Vec::new();
let mut current_cursor = cursor;
for _ in 0..iterations {
let raw: RedisRawValue = redis::cmd("SCAN")
.arg(current_cursor)
.arg("MATCH")
.arg(pattern)
.arg("COUNT")
.arg(count)
.query_async(con)
.await
.map_err(|e| e.to_string())?;
let (next_cursor, keys) = parse_scan_keys(raw)?;
if !keys.is_empty() {
let mut pipe = redis::pipe();
for key in &keys {
pipe.cmd("TYPE").arg(key);
}
let key_types: Vec<String> = pipe.query_async(con).await.unwrap_or_default();
for (index, key) in keys.iter().enumerate() {
let key_type = key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string());
all_keys.push(RedisKeyInfo {
key_display: redis_key_bytes_to_display(key),
key_raw: redis_key_bytes_to_raw(key),
key_type,
ttl: -2,
size: 0,
value_preview: redis_key_value_preview(
key_types.get(index).map(String::as_str).unwrap_or("unknown"),
),
});
}
}
if next_cursor == 0 {
return Ok(RedisScanResult { cursor: 0, keys: all_keys, total_keys });
}
current_cursor = next_cursor;
}
let mut pipe = redis::pipe();
for key in &keys {
pipe.cmd("TYPE").arg(key);
}
let key_types: Vec<String> = pipe.query_async(con).await.unwrap_or_default();
let mut result = Vec::with_capacity(keys.len());
for (index, key) in keys.iter().enumerate() {
let key_type = key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string());
result.push(RedisKeyInfo {
key_display: redis_key_bytes_to_display(key),
key_raw: redis_key_bytes_to_raw(key),
key_type,
ttl: -2,
size: 0,
value_preview: redis_key_value_preview(key_types.get(index).map(String::as_str).unwrap_or("unknown")),
});
}
Ok(RedisScanResult { cursor: next_cursor, keys: result, total_keys })
Ok(RedisScanResult { cursor: current_cursor, keys: all_keys, total_keys })
}
pub async fn scan_values_page<C>(
@ -1048,7 +1079,8 @@ pub async fn scan_values_page<C>(
where
C: ConnectionLike + Send + Sync + Unpin,
{
let total_keys: u64 = redis::cmd("DBSIZE").query_async(con).await.unwrap_or(0);
// Only call DBSIZE on the first page (cursor == 0) to avoid redundant work.
let total_keys: u64 = if cursor == 0 { redis::cmd("DBSIZE").query_async(con).await.unwrap_or(0) } else { 0 };
if query.trim().is_empty() {
return Ok(RedisScanResult { cursor, keys: Vec::new(), total_keys });
}

42
crates/dbx-core/src/redis_ops.rs
View File

@ -1,6 +1,6 @@
use crate::connection::{AppState, PoolKind};
use crate::db::redis_driver::{
self, RedisCommandResult, RedisConnection, RedisDatabaseInfo, RedisScanResult, RedisValue,
self, RedisCommandResult, RedisConnection, RedisDatabaseInfo, RedisKeyInfo, RedisScanResult, RedisValue,
};
async fn ensure_redis_pool(state: &AppState, connection_id: &str) -> Result<(), String> {
@ -33,6 +33,23 @@ pub async fn redis_scan_keys_core(
cursor: u64,
pattern: &str,
count: usize,
) -> Result<RedisScanResult, String> {
redis_scan_keys_batch_core(state, connection_id, db, cursor, pattern, count, 1).await
}
/// Batch-scan keys with server-side multi-SCAN support.
///
/// Performs up to `max_iterations` SCAN→TYPE cycles server-side in a single
/// API call, dramatically reducing frontend↔backend roundtrips when fetching
/// many keys (e.g. "fetch all" in the key browser).
pub async fn redis_scan_keys_batch_core(
state: &AppState,
connection_id: &str,
db: u32,
cursor: u64,
pattern: &str,
count: usize,
max_iterations: usize,
) -> Result<RedisScanResult, String> {
ensure_redis_pool(state, connection_id).await?;
let connections = state.connections.read().await;
@ -42,11 +59,30 @@ pub async fn redis_scan_keys_core(
RedisConnection::Direct(con) => {
let mut con = con.lock().await;
redis_driver::select_db(&mut *con, db).await?;
redis_driver::scan_keys_page(&mut *con, cursor, pattern, count).await
redis_driver::scan_keys_batch(&mut *con, cursor, pattern, count, max_iterations).await
}
RedisConnection::Cluster(cluster) => {
redis_driver::ensure_cluster_db(db)?;
redis_driver::scan_cluster_keys_page(cluster, cursor, pattern, count).await
// Cluster scan already iterates across nodes; for batch mode we
// loop the cluster-level scan to accumulate keys server-side.
if max_iterations <= 1 {
return redis_driver::scan_cluster_keys_page(cluster, cursor, pattern, count).await;
}
let mut all_keys: Vec<RedisKeyInfo> = Vec::new();
let mut current_cursor = cursor;
let mut total_keys: u64 = 0;
for i in 0..max_iterations {
let page = redis_driver::scan_cluster_keys_page(cluster, current_cursor, pattern, count).await?;
if i == 0 {
total_keys = page.total_keys;
}
all_keys.extend(page.keys);
if page.cursor == 0 {
return Ok(RedisScanResult { cursor: 0, keys: all_keys, total_keys });
}
current_cursor = page.cursor;
}
Ok(RedisScanResult { cursor: current_cursor, keys: all_keys, total_keys })
}
},
_ => Err("Not a Redis connection".to_string()),

1
crates/dbx-web/src/main.rs
View File

@ -240,6 +240,7 @@ async fn main() {
// Redis
.route("/redis/list-databases", post(routes::redis::list_databases))
.route("/redis/scan-keys", post(routes::redis::scan_keys))
.route("/redis/scan-keys-batch", post(routes::redis::scan_keys_batch))
.route("/redis/scan-values", post(routes::redis::scan_values))
.route("/redis/get-value", post(routes::redis::get_value))
.route("/redis/set-string", post(routes::redis::set_string))

34
crates/dbx-web/src/routes/redis.rs
View File

@ -38,6 +38,22 @@ pub struct RedisScanRequest {
pub count: usize,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RedisScanBatchRequest {
pub connection_id: String,
pub db: u32,
pub cursor: u64,
pub pattern: String,
pub count: usize,
#[serde(default = "default_max_iterations")]
pub max_iterations: usize,
}
fn default_max_iterations() -> usize {
1
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RedisValueScanRequest {
@ -191,6 +207,24 @@ pub async fn scan_keys(
Ok(Json(serde_json::to_value(result).map_err(|e| AppError(e.to_string()))?))
}
pub async fn scan_keys_batch(
State(state): State<Arc<WebState>>,
Json(req): Json<RedisScanBatchRequest>,
) -> Result<Json<serde_json::Value>, AppError> {
let result = dbx_core::redis_ops::redis_scan_keys_batch_core(
&state.app,
&req.connection_id,
req.db,
req.cursor,
&req.pattern,
req.count,
req.max_iterations,
)
.await
.map_err(AppError)?;
Ok(Json(serde_json::to_value(result).map_err(|e| AppError(e.to_string()))?))
}
pub async fn scan_values(
State(state): State<Arc<WebState>>,
Json(req): Json<RedisValueScanRequest>,

14
src-tauri/src/commands/redis_cmd.rs
View File

@ -26,6 +26,20 @@ pub async fn redis_scan_keys(
dbx_core::redis_ops::redis_scan_keys_core(&state, &connection_id, db, cursor, &pattern, count).await
}
#[tauri::command]
pub async fn redis_scan_keys_batch(
state: State<'_, Arc<AppState>>,
connection_id: String,
db: u32,
cursor: u64,
pattern: String,
count: usize,
max_iterations: usize,
) -> Result<RedisScanResult, String> {
dbx_core::redis_ops::redis_scan_keys_batch_core(&state, &connection_id, db, cursor, &pattern, count, max_iterations)
.await
}
#[tauri::command]
pub async fn redis_scan_values(
state: State<'_, Arc<AppState>>,

1
src-tauri/src/lib.rs
View File

@ -464,6 +464,7 @@ pub fn run() {
commands::table_import::cancel_table_import,
commands::redis_cmd::redis_list_databases,
commands::redis_cmd::redis_scan_keys,
commands::redis_cmd::redis_scan_keys_batch,
commands::redis_cmd::redis_scan_values,
commands::redis_cmd::redis_get_value,
commands::redis_cmd::redis_set_string,