xiaoxue_admin
/
dbx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(redis): speed up large key scans

This commit is contained in:
t8y2 2026-06-23 15:13:04 +08:00
parent ff79ec05ad
commit fc9f97b6b5
9 changed files with 154 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
apps/desktop/src/components/redis/RedisKeyBrowser.vue
View File

@ -18,7 +18,7 @@ import RedisValueViewer from "./RedisValueViewer.vue";
import RedisPubSubPanel from "./RedisPubSubPanel.vue";
import RedisSlowlogPanel from "./RedisSlowlogPanel.vue";
import * as api from "@/lib/api";
import type { RedisKeyInfo, RedisScanResult, HistoryEntry } from "@/lib/api";
import type { RedisKeyInfo, RedisScanResult, RedisValue, HistoryEntry } from "@/lib/api";
import { uuid } from "@/lib/utils";
import { useConnectionStore } from "@/stores/connectionStore";
import { useSettingsStore } from "@/stores/settingsStore";
@ -102,6 +102,7 @@ let nextEntryId = 0;
let searchRequestId = 0;
let redisBrowserIsActive = true;
let redisDbFlushedListenerRegistered = false;
const loadedKeyRaws = new Set<string>();
const valueQuery = computed(() => searchPattern.value.trim());
const isValueSearchMode = computed(() => searchMode.value === "value" || searchMode.value === "all");
@ -200,24 +201,30 @@ function mergeTree(newKeys: RedisKeyInfo[]) {
async function fetchScanPage(): Promise<RedisScanResult> {
const pageSize = settingsStore.editorSettings.redisScanPageSize;
return isValueSearchMode.value ? await api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all") : await api.redisScanKeys(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize);
return isValueSearchMode.value ? await api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all") : await api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, 1, false);
}
/// Batch-scan variant that performs multiple SCAN iterations server-side.
/// Dramatically reduces frontend↔backend roundtrips for bulk loading.
async function fetchScanBatchPage(maxIterations: number): Promise<RedisScanResult> {
const pageSize = settingsStore.editorSettings.redisScanPageSize;
async function fetchScanBatchPage(maxIterations: number, options: { count?: number; includeTypes?: boolean } = {}): Promise<RedisScanResult> {
const pageSize = options.count ?? settingsStore.editorSettings.redisScanPageSize;
// Value search cannot be batched because each key requires a GET.
if (isValueSearchMode.value) {
return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all");
}
return api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, maxIterations);
return api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, maxIterations, options.includeTypes ?? false);
}
function appendScanResult(result: RedisScanResult) {
const existingKeys = new Set(flatKeys.value.map((key) => key.key_raw));
const newKeys = result.keys.filter((key) => !existingKeys.has(key.key_raw));
flatKeys.value = [...flatKeys.value, ...newKeys];
function appendScanResult(result: RedisScanResult, options: { updateTree?: boolean } = {}) {
const newKeys: RedisKeyInfo[] = [];
for (const key of result.keys) {
if (loadedKeyRaws.has(key.key_raw)) continue;
loadedKeyRaws.add(key.key_raw);
newKeys.push(key);
}
if (newKeys.length > 0) {
flatKeys.value = [...flatKeys.value, ...newKeys];
}
scanCursor.value = result.cursor;
hasMore.value = result.cursor !== 0;
// DBSIZE is only called on the first batch page (cursor==0); subsequent
@ -228,10 +235,12 @@ function appendScanResult(result: RedisScanResult) {
lastTotalKeys.value = result.total_keys;
}
if (treeKeys.value.length === 0) {
rebuildTree(isSearchMode.value);
} else {
mergeTree(newKeys);
if (options.updateTree ?? true) {
if (treeKeys.value.length === 0) {
rebuildTree(isSearchMode.value);
} else {
mergeTree(newKeys);
}
}
connectionStore.updateRedisDbKeyStats(props.connectionId, props.db, {
@ -276,6 +285,7 @@ async function loadKeys() {
const requestId = ++searchRequestId;
isFetchingAll.value = false;
loading.value = true;
loadedKeyRaws.clear();
flatKeys.value = [];
treeKeys.value = [];
selectedKeyRaw.value = null;
@ -313,25 +323,29 @@ async function loadMore() {
}
}
/// Fetch-all with server-side multi-SCAN batching.
///
/// Each call performs up to 15 SCAN→TYPE cycles server-side (~0.5s per
/// batch at COUNT=1000). This keeps the UI responsive with frequent progress
/// updates while still avoiding the per-page overhead of single-SCAN calls.
const FETCH_ALL_BATCH_ITERATIONS = 15;
// Fetch-all uses large key-only SCAN pages and rebuilds the tree once at the
// end; per-page tree sorting dominates runtime on million-key pattern scans.
const FETCH_ALL_SCAN_COUNT = 50000;
const FETCH_ALL_BATCH_ITERATIONS = 1;
async function fetchAll() {
if (!hasMore.value || isFetchingAll.value) return;
const requestId = searchRequestId;
isFetchingAll.value = true;
let changed = false;
try {
while (requestId === searchRequestId && isFetchingAll.value && hasMore.value) {
const result = await fetchScanBatchPage(FETCH_ALL_BATCH_ITERATIONS);
const result = await fetchScanBatchPage(FETCH_ALL_BATCH_ITERATIONS, {
count: FETCH_ALL_SCAN_COUNT,
includeTypes: false,
});
if (requestId !== searchRequestId) break;
appendScanResult(result);
appendScanResult(result, { updateTree: false });
changed = true;
}
} finally {
if (requestId === searchRequestId) {
if (changed) rebuildTree(isSearchMode.value);
isFetchingAll.value = false;
}
}
@ -360,6 +374,7 @@ function onRowClick(node: RedisKeyTreeNode) {
function onKeyDeleted() {
if (!selectedKeyRaw.value) return;
loadedKeyRaws.delete(selectedKeyRaw.value);
flatKeys.value = flatKeys.value.filter((key) => key.key_raw !== selectedKeyRaw.value);
selectedKeyRaw.value = null;
rebuildTree(false);
@ -369,6 +384,26 @@ function onKeyDeleted() {
});
}
function redisValueToKeyInfo(value: RedisValue): RedisKeyInfo {
return {
key_display: value.key_display,
key_raw: value.key_raw,
key_type: value.key_type,
ttl: value.ttl,
size: typeof value.value === "string" ? value.value.length : (value.total ?? 0),
value_preview: createdKeyPreview(value.value),
};
}
function onKeyLoaded(value: RedisValue) {
const keyInfo = redisValueToKeyInfo(value);
const existingIndex = flatKeys.value.findIndex((key) => key.key_raw === keyInfo.key_raw);
if (existingIndex < 0) return;
flatKeys.value = flatKeys.value.map((key, index) => (index === existingIndex ? keyInfo : key));
loadedKeyRaws.add(keyInfo.key_raw);
rebuildTree(false);
}
function toggleCheck(keyRaw: string, event: Event) {
event.stopPropagation();
const next = new Set(checkedKeys.value);
@ -393,6 +428,7 @@ function requestGroupDelete(node: RedisKeyTreeNode, event: Event) {
}
function resetLoadedKeys() {
loadedKeyRaws.clear();
flatKeys.value = [];
treeKeys.value = [];
selectedKeyRaw.value = null;
@ -404,6 +440,7 @@ function resetLoadedKeys() {
async function deleteKeyRaws(keys: string[]) {
const deletedCount = await api.redisDeleteKeys(props.connectionId, props.db, keys);
const deleted = new Set(keys);
for (const key of deleted) loadedKeyRaws.delete(key);
flatKeys.value = flatKeys.value.filter((k) => !deleted.has(k.key_raw));
if (selectedKeyRaw.value && deleted.has(selectedKeyRaw.value)) {
selectedKeyRaw.value = null;
@ -601,6 +638,7 @@ function upsertCreatedKey(value: any) {
} else {
flatKeys.value = [keyInfo, ...flatKeys.value];
}
loadedKeyRaws.add(keyInfo.key_raw);
selectedKeyRaw.value = keyInfo.key_raw;
rebuildTree(isSearchMode.value);
connectionStore.updateRedisDbKeyStats(props.connectionId, props.db, {
@ -974,7 +1012,7 @@ defineExpose({ focusSearch });
</div>
<div class="flex shrink-0 items-center justify-end gap-1">
<Badge v-if="row.node.kind === 'leaf'" variant="outline" class="text-xs px-1.5 py-0" :class="typeColor(row.node.keyType)">{{ row.node.keyType }}</Badge>
<Badge v-if="row.node.kind === 'leaf' && row.node.keyType !== 'unknown'" variant="outline" class="text-xs px-1.5 py-0" :class="typeColor(row.node.keyType)">{{ row.node.keyType }}</Badge>
<Button v-if="row.node.kind === 'group'" variant="ghost" size="icon" class="h-5 w-5 shrink-0 text-destructive opacity-0 group-hover:opacity-100" :title="t('redis.deleteGroup')" @click="requestGroupDelete(row.node, $event)">
<Trash2 class="h-3 w-3" />
</Button>
@ -1031,7 +1069,7 @@ defineExpose({ focusSearch });
</div>
<TabsContent value="detail" class="m-0 min-h-0 flex-1 flex flex-col">
<RedisValueViewer v-if="selectedKey" :key="selectedKey.key_raw" :connection-id="connectionId" :db="db" :key-display="selectedKey.key_display" :key-raw="selectedKey.key_raw" :metadata="selectedKey" @deleted="onKeyDeleted" />
<RedisValueViewer v-if="selectedKey" :key="selectedKey.key_raw" :connection-id="connectionId" :db="db" :key-display="selectedKey.key_display" :key-raw="selectedKey.key_raw" :metadata="selectedKey" @deleted="onKeyDeleted" @loaded="onKeyLoaded" />
<div v-else class="flex-1 flex items-center justify-center text-xs text-muted-foreground">
{{ t("redis.selectKeyForDetail") }}
</div>

6
apps/desktop/src/components/redis/RedisValueViewer.vue
View File

@ -32,7 +32,7 @@ const props = defineProps<{
metadata?: RedisKeyInfo | null;
}>();
const emit = defineEmits<{ deleted: [] }>();
const emit = defineEmits<{ deleted: []; loaded: [value: RedisValue] }>();
const data = ref<RedisValue | null>(null);
const loading = ref(false);
@ -199,7 +199,9 @@ async function load(options: { selectDefaultMember?: boolean } = {}) {
const shouldSelectDefaultMember = options.selectDefaultMember ?? true;
loading.value = true;
try {
data.value = await api.redisGetValue(props.connectionId, props.db, props.keyRaw);
const loadedValue = await api.redisGetValue(props.connectionId, props.db, props.keyRaw);
data.value = loadedValue;
emit("loaded", loadedValue);
scanCursor.value = data.value.scan_cursor ?? undefined;
if (data.value.key_type === "string") {
const detail = formatRedisMemberDetail(data.value.value);

4
apps/desktop/src/lib/http.ts
View File

@ -1366,8 +1366,8 @@ export async function redisScanKeys(connectionId: string, db: number, cursor: nu
return post("/api/redis/scan-keys", { connectionId, db, cursor, pattern, count });
}
export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number): Promise<RedisScanResult> {
return post("/api/redis/scan-keys-batch", { connectionId, db, cursor, pattern, count, maxIterations });
export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number, includeTypes = true): Promise<RedisScanResult> {
return post("/api/redis/scan-keys-batch", { connectionId, db, cursor, pattern, count, maxIterations, includeTypes });
}
export async function redisScanValues(connectionId: string, db: number, cursor: number, pattern: string, query: string, count: number, includeKeyMatches = false): Promise<RedisScanResult> {

4
apps/desktop/src/lib/tauri.ts
View File

@ -1163,8 +1163,8 @@ export async function redisScanKeys(connectionId: string, db: number, cursor: nu
return invoke("redis_scan_keys", { connectionId, db, cursor, pattern, count });
}
export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number): Promise<RedisScanResult> {
return invoke("redis_scan_keys_batch", { connectionId, db, cursor, pattern, count, maxIterations });
export async function redisScanKeysBatch(connectionId: string, db: number, cursor: number, pattern: string, count: number, maxIterations: number, includeTypes = true): Promise<RedisScanResult> {
return invoke("redis_scan_keys_batch", { connectionId, db, cursor, pattern, count, maxIterations, includeTypes });
}
export async function redisScanValues(connectionId: string, db: number, cursor: number, pattern: string, query: string, count: number, includeKeyMatches = false): Promise<RedisScanResult> {

2
apps/desktop/src/stores/connectionStore.ts
View File

@ -2358,7 +2358,7 @@ export const useConnectionStore = defineStore("connection", () => {
await ensureConnected(connectionId);
const pageSize = settingsStore.editorSettings.redisScanPageSize;
// Bounded multi-round SCAN: trade coverage for latency/memory safety.
const result = await api.redisScanKeysBatch(connectionId, Number(database), 0, "*", pageSize, 6);
const result = await api.redisScanKeysBatch(connectionId, Number(database), 0, "*", pageSize, 6, false);
const keys = result.keys.map((key) => key.key_display).slice(0, REDIS_COMPLETION_KEYS_MAX);
redisCompletionKeysCache.value[cacheKey] = keys;
evictOldestCacheEntries(redisCompletionKeysCache.value, COMPLETION_CACHE_MAX);

61
crates/dbx-core/src/db/redis_driver.rs
View File

@ -760,6 +760,16 @@ pub async fn scan_cluster_keys_page(
cursor: u64,
pattern: &str,
count: usize,
) -> Result<RedisScanResult, String> {
scan_cluster_keys_page_with_options(pool, cursor, pattern, count, true).await
}
pub async fn scan_cluster_keys_page_with_options(
pool: &RedisClusterPool,
cursor: u64,
pattern: &str,
count: usize,
include_types: bool,
) -> Result<RedisScanResult, String> {
let master_nodes = cluster_master_nodes(pool).await?;
if master_nodes.is_empty() {
@ -776,7 +786,7 @@ pub async fn scan_cluster_keys_page(
let endpoint = &master_nodes[index];
let mut con = connect_cluster_node(pool, endpoint).await?;
let current_cursor = if index == node_index { node_cursor } else { 0 };
let result = scan_keys_page(&mut con, current_cursor, pattern, count).await?;
let result = scan_keys_page_with_options(&mut con, current_cursor, pattern, count, include_types).await?;
if !result.keys.is_empty() {
let next_cursor = if result.cursor != 0 {
encode_cluster_cursor(index, result.cursor)?
@ -1434,13 +1444,26 @@ pub async fn scan_keys_page<C>(con: &mut C, cursor: u64, pattern: &str, count: u
where
C: ConnectionLike + Send + Sync + Unpin,
{
scan_keys_batch(con, cursor, pattern, count, 1).await
scan_keys_batch(con, cursor, pattern, count, 1, true).await
}
pub async fn scan_keys_page_with_options<C>(
con: &mut C,
cursor: u64,
pattern: &str,
count: usize,
include_types: bool,
) -> Result<RedisScanResult, String>
where
C: ConnectionLike + Send + Sync + Unpin,
{
scan_keys_batch(con, cursor, pattern, count, 1, include_types).await
}
/// Batch-scan keys with server-side multi-SCAN support.
///
/// Performs up to `max_iterations` SCAN→TYPE cycles in a single call,
/// dramatically reducing frontend↔backend roundtrips when fetching many keys.
/// Performs up to `max_iterations` SCAN cycles in a single call. TYPE metadata
/// is optional so large key-name searches can avoid extra Redis work.
/// DBSIZE is only called on the first iteration (cursor == 0).
pub async fn scan_keys_batch<C>(
con: &mut C,
@ -1448,6 +1471,7 @@ pub async fn scan_keys_batch<C>(
pattern: &str,
count: usize,
max_iterations: usize,
include_types: bool,
) -> Result<RedisScanResult, String>
where
C: ConnectionLike + Send + Sync + Unpin,
@ -1472,23 +1496,34 @@ where
let (next_cursor, keys) = parse_scan_keys(raw)?;
if !keys.is_empty() {
let mut pipe = redis::pipe();
for key in &keys {
pipe.cmd("TYPE").arg(key);
}
let key_types: Vec<String> = pipe.query_async(con).await.unwrap_or_default();
let key_types: Vec<String> = if include_types {
let mut pipe = redis::pipe();
for key in &keys {
pipe.cmd("TYPE").arg(key);
}
pipe.query_async(con).await.unwrap_or_default()
} else {
Vec::new()
};
for (index, key) in keys.iter().enumerate() {
let key_type = key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string());
let key_type = if include_types {
key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string())
} else {
"unknown".to_string()
};
let value_preview = if include_types {
redis_key_value_preview(key_types.get(index).map(String::as_str).unwrap_or("unknown"))
} else {
String::new()
};
all_keys.push(RedisKeyInfo {
key_display: redis_key_bytes_to_display(key),
key_raw: redis_key_bytes_to_raw(key),
key_type,
ttl: -2,
size: 0,
value_preview: redis_key_value_preview(
key_types.get(index).map(String::as_str).unwrap_or("unknown"),
),
value_preview,
});
}
}

29
crates/dbx-core/src/redis_ops.rs
View File

@ -34,14 +34,14 @@ pub async fn redis_scan_keys_core(
pattern: &str,
count: usize,
) -> Result<RedisScanResult, String> {
redis_scan_keys_batch_core(state, connection_id, db, cursor, pattern, count, 1).await
redis_scan_keys_batch_core(state, connection_id, db, cursor, pattern, count, 1, true).await
}
/// Batch-scan keys with server-side multi-SCAN support.
///
/// Performs up to `max_iterations` SCAN→TYPE cycles server-side in a single
/// API call, dramatically reducing frontend↔backend roundtrips when fetching
/// many keys (e.g. "fetch all" in the key browser).
/// Performs up to `max_iterations` SCAN cycles server-side in a single API
/// call, dramatically reducing frontend↔backend roundtrips when fetching many
/// keys (e.g. "fetch all" in the key browser). TYPE metadata is optional.
pub async fn redis_scan_keys_batch_core(
state: &AppState,
connection_id: &str,
@ -50,6 +50,7 @@ pub async fn redis_scan_keys_batch_core(
pattern: &str,
count: usize,
max_iterations: usize,
include_types: bool,
) -> Result<RedisScanResult, String> {
ensure_redis_pool(state, connection_id).await?;
let connections = state.connections.read().await;
@ -59,20 +60,34 @@ pub async fn redis_scan_keys_batch_core(
RedisConnection::Direct(con) => {
let mut con = con.lock().await;
redis_driver::select_db(&mut *con, db).await?;
redis_driver::scan_keys_batch(&mut *con, cursor, pattern, count, max_iterations).await
redis_driver::scan_keys_batch(&mut *con, cursor, pattern, count, max_iterations, include_types).await
}
RedisConnection::Cluster(cluster) => {
redis_driver::ensure_cluster_db(db)?;
// Cluster scan already iterates across nodes; for batch mode we
// loop the cluster-level scan to accumulate keys server-side.
if max_iterations <= 1 {
return redis_driver::scan_cluster_keys_page(cluster, cursor, pattern, count).await;
return redis_driver::scan_cluster_keys_page_with_options(
cluster,
cursor,
pattern,
count,
include_types,
)
.await;
}
let mut all_keys: Vec<RedisKeyInfo> = Vec::new();
let mut current_cursor = cursor;
let mut total_keys: u64 = 0;
for i in 0..max_iterations {
let page = redis_driver::scan_cluster_keys_page(cluster, current_cursor, pattern, count).await?;
let page = redis_driver::scan_cluster_keys_page_with_options(
cluster,
current_cursor,
pattern,
count,
include_types,
)
.await?;
if i == 0 {
total_keys = page.total_keys;
}

2
crates/dbx-web/src/routes/redis.rs
View File

@ -48,6 +48,7 @@ pub struct RedisScanBatchRequest {
pub count: usize,
#[serde(default = "default_max_iterations")]
pub max_iterations: usize,
pub include_types: Option<bool>,
}
fn default_max_iterations() -> usize {
@ -234,6 +235,7 @@ pub async fn scan_keys_batch(
&req.pattern,
req.count,
req.max_iterations,
req.include_types.unwrap_or(true),
)
.await
.map_err(AppError)?;

14
src-tauri/src/commands/redis_cmd.rs
View File

@ -35,9 +35,19 @@ pub async fn redis_scan_keys_batch(
pattern: String,
count: usize,
max_iterations: usize,
include_types: Option<bool>,
) -> Result<RedisScanResult, String> {
dbx_core::redis_ops::redis_scan_keys_batch_core(&state, &connection_id, db, cursor, &pattern, count, max_iterations)
.await
dbx_core::redis_ops::redis_scan_keys_batch_core(
&state,
&connection_id,
db,
cursor,
&pattern,
count,
max_iterations,
include_types.unwrap_or(true),
)
.await
}
#[tauri::command]