diff --git a/apps/desktop/src/components/redis/RedisKeyBrowser.vue b/apps/desktop/src/components/redis/RedisKeyBrowser.vue index 857d234b7..03aaa03e5 100644 --- a/apps/desktop/src/components/redis/RedisKeyBrowser.vue +++ b/apps/desktop/src/components/redis/RedisKeyBrowser.vue @@ -22,7 +22,7 @@ import type { RedisKeyInfo, RedisScanResult, RedisValue, HistoryEntry } from "@/ import { uuid } from "@/lib/utils"; import { useConnectionStore } from "@/stores/connectionStore"; import { useSettingsStore } from "@/stores/settingsStore"; -import { buildRedisKeyTree, collectExpandedGroupIds, collectRedisGroupKeyRaws, flattenVisibleRedisKeyTree, mergeKeysIntoRedisKeyTree, type RedisKeyTreeNode } from "@/lib/redisKeyTree"; +import { buildRedisKeyTree, collectExpandedGroupIds, collectRedisGroupKeyRaws, flattenVisibleRedisKeyTree, mergeKeysIntoRedisKeyTree, redisKeyToFlatTreeRow, type RedisKeyTreeNode } from "@/lib/redisKeyTree"; import { classifyRedisCommandSafety } from "@/lib/redisCommandSafety"; import { isRedisMutatingCommand } from "@/lib/redisCommandTable"; import { isRedisClearScreenCommand, nextRedisCommandDb, redisKeyTextToRaw } from "@/lib/redisCommandSession"; @@ -108,6 +108,7 @@ const valueQuery = computed(() => searchPattern.value.trim()); const isValueSearchMode = computed(() => searchMode.value === "value" || searchMode.value === "all"); const effectivePattern = computed(() => (searchMode.value === "key" ? redisKeySearchPattern(searchPattern.value, fuzzyKeySearch.value) : "*")); const isSearchMode = computed(() => (searchMode.value === "key" ? effectivePattern.value !== "*" : valueQuery.value !== "")); +const useFlatKeySearchRows = computed(() => searchMode.value === "key" && isSearchMode.value); const searchPlaceholder = computed(() => { if (searchMode.value === "key") return fuzzyKeySearch.value ? t("redis.fuzzyPattern") : t("redis.pattern"); return searchMode.value === "all" ? t("redis.allSearchPlaceholder") : t("redis.valueSearchPlaceholder"); @@ -150,12 +151,13 @@ const createKeyTypeOptions = computed<{ value: RedisCreateKeyType; label: string { value: "stream", label: "Stream" }, { value: "json", label: "JSON" }, ]); -const visibleRows = computed(() => - flattenVisibleRedisKeyTree(treeKeys.value, expandedGroupIds.value).map((row) => ({ +const visibleRows = computed(() => { + const rows = useFlatKeySearchRows.value ? flatKeys.value.map((key) => redisKeyToFlatTreeRow(key, props.db)) : flattenVisibleRedisKeyTree(treeKeys.value, expandedGroupIds.value); + return rows.map((row) => ({ ...row, id: row.node.id, - })), -); + })); +}); let commandHistoryId = 0; function countLeaves(node: RedisKeyTreeNode): number { @@ -236,7 +238,10 @@ function appendScanResult(result: RedisScanResult, options: { updateTree?: boole } if (options.updateTree ?? true) { - if (treeKeys.value.length === 0) { + if (useFlatKeySearchRows.value) { + treeKeys.value = []; + expandedGroupIds.value = new Set(); + } else if (treeKeys.value.length === 0) { rebuildTree(isSearchMode.value); } else { mergeTree(newKeys); @@ -345,7 +350,7 @@ async function fetchAll() { } } finally { if (requestId === searchRequestId) { - if (changed) rebuildTree(isSearchMode.value); + if (changed && !useFlatKeySearchRows.value) rebuildTree(isSearchMode.value); isFetchingAll.value = false; } } @@ -1012,7 +1017,7 @@ defineExpose({ focusSearch });
- {{ row.node.keyType }} + {{ row.node.keyType }} diff --git a/apps/desktop/src/components/redis/RedisValueViewer.vue b/apps/desktop/src/components/redis/RedisValueViewer.vue index 1e24e43d3..346a5bac1 100644 --- a/apps/desktop/src/components/redis/RedisValueViewer.vue +++ b/apps/desktop/src/components/redis/RedisValueViewer.vue @@ -182,12 +182,13 @@ const isBinaryStringValue = computed(() => data.value?.key_type === "string" && const hasMore = computed(() => scanCursor.value != null && scanCursor.value > 0); const metadataSizeLabel = computed(() => { const metadata = props.metadata; - if (!metadata || metadata.size <= 0) return ""; + const size = metadata?.size ?? 0; + if (!metadata || size <= 0) return ""; if (metadata.key_type === "string") { - if (metadata.size >= 1024) return `${(metadata.size / 1024).toFixed(1)} KB`; - return `${metadata.size} B`; + if (size >= 1024) return `${(size / 1024).toFixed(1)} KB`; + return `${size} B`; } - return String(metadata.size); + return String(size); }); function collectionCountLabel(kind: "items" | "fields" | "members", loaded: number, total?: number | null) { diff --git a/apps/desktop/src/lib/redisKeyTree.ts b/apps/desktop/src/lib/redisKeyTree.ts index 37b24d271..c76a0f78c 100644 --- a/apps/desktop/src/lib/redisKeyTree.ts +++ b/apps/desktop/src/lib/redisKeyTree.ts @@ -37,6 +37,25 @@ function buildLeafId(db: number, keyRaw: string): string { return `leaf:${db}:${keyRaw}`; } +export function redisKeyToFlatTreeRow(key: RedisKeyInfo, db: number): RedisKeyTreeRow { + return { + node: { + kind: "leaf", + id: buildLeafId(db, key.key_raw), + label: key.key_display, + fullKeyDisplay: key.key_display, + keyRaw: key.key_raw, + db, + keyType: key.key_type ?? "", + ttl: key.ttl ?? -2, + size: key.size ?? 0, + valuePreview: key.value_preview ?? "", + pathSegments: [key.key_display], + }, + depth: 0, + }; +} + function compareRedisTreeNodes(a: RedisKeyTreeNode, b: RedisKeyTreeNode): number { if (a.kind !== b.kind) return a.kind === "group" ? -1 : 1; return a.label.localeCompare(b.label); @@ -74,10 +93,10 @@ function insertKeyIntoTree(root: RedisKeyTreeNode[], groupMap: Map, depth = 0): RedisKeyTreeRow[] { const rows: RedisKeyTreeRow[] = []; + const stack: RedisKeyTreeRow[] = []; - for (const node of nodes) { - rows.push({ node, depth }); - if (node.kind === "group" && expandedGroupIds.has(node.id)) { - rows.push(...flattenVisibleRedisKeyTree(node.children, expandedGroupIds, depth + 1)); + for (let index = nodes.length - 1; index >= 0; index--) { + stack.push({ node: nodes[index], depth }); + } + + while (stack.length > 0) { + const row = stack.pop()!; + rows.push(row); + + if (row.node.kind !== "group" || !expandedGroupIds.has(row.node.id)) continue; + + const childDepth = row.depth + 1; + for (let index = row.node.children.length - 1; index >= 0; index--) { + stack.push({ node: row.node.children[index], depth: childDepth }); } } diff --git a/apps/desktop/src/lib/tauri.ts b/apps/desktop/src/lib/tauri.ts index 558c83139..91f32ba43 100644 --- a/apps/desktop/src/lib/tauri.ts +++ b/apps/desktop/src/lib/tauri.ts @@ -1113,10 +1113,10 @@ export async function getAppVersion(): Promise { export interface RedisKeyInfo { key_display: string; key_raw: string; - key_type: string; - ttl: number; - size: number; - value_preview: string; + key_type?: string; + ttl?: number; + size?: number; + value_preview?: string; } export interface RedisDatabaseInfo { diff --git a/crates/dbx-core/src/db/redis_driver.rs b/crates/dbx-core/src/db/redis_driver.rs index 68674ee44..d482124d3 100644 --- a/crates/dbx-core/src/db/redis_driver.rs +++ b/crates/dbx-core/src/db/redis_driver.rs @@ -30,12 +30,28 @@ pub struct RedisDatabaseInfo { pub struct RedisKeyInfo { pub key_display: String, pub key_raw: String, + #[serde(default, skip_serializing_if = "String::is_empty")] pub key_type: String, + #[serde(default = "default_missing_ttl", skip_serializing_if = "is_missing_ttl")] pub ttl: i64, + #[serde(default, skip_serializing_if = "is_zero_u64")] pub size: u64, + #[serde(default, skip_serializing_if = "String::is_empty")] pub value_preview: String, } +fn default_missing_ttl() -> i64 { + -2 +} + +fn is_missing_ttl(ttl: &i64) -> bool { + *ttl == -2 +} + +fn is_zero_u64(value: &u64) -> bool { + *value == 0 +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RedisScanResult { pub cursor: u64, @@ -1510,7 +1526,7 @@ where let key_type = if include_types { key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string()) } else { - "unknown".to_string() + String::new() }; let value_preview = if include_types { redis_key_value_preview(key_types.get(index).map(String::as_str).unwrap_or("unknown")) diff --git a/package.json b/package.json index 339d5037b..6bdc4ba30 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "build": "vite build --config apps/desktop/vite.config.ts", "build:checked": "pnpm typecheck && pnpm build", "check": "node scripts/run-check.mjs", + "bench:redis-key-search": "node scripts/bench/redis-key-search.mjs", "lint": "oxlint --vue-plugin apps/desktop/src", "fmt": "oxfmt \"apps/desktop/src/**/*.{ts,vue}\"", "test": "vitest run", diff --git a/packages/app-tests/redisKeyTree.test.ts b/packages/app-tests/redisKeyTree.test.ts index b298d6652..5d77e54d0 100644 --- a/packages/app-tests/redisKeyTree.test.ts +++ b/packages/app-tests/redisKeyTree.test.ts @@ -1,6 +1,6 @@ import { test } from "vitest"; import assert from "node:assert/strict"; -import { buildRedisKeyTree, collectRedisGroupKeyRaws, collectExpandedGroupIds, flattenVisibleRedisKeyTree, mergeKeysIntoRedisKeyTree, type RedisKeyTreeNode } from "../../apps/desktop/src/lib/redisKeyTree.ts"; +import { buildRedisKeyTree, collectRedisGroupKeyRaws, collectExpandedGroupIds, flattenVisibleRedisKeyTree, mergeKeysIntoRedisKeyTree, redisKeyToFlatTreeRow, type RedisKeyTreeNode } from "../../apps/desktop/src/lib/redisKeyTree.ts"; import type { RedisKeyInfo } from "../../apps/desktop/src/lib/api.ts"; function makeKey(key_display: string, key_raw: string, key_type = "string", ttl = -1): RedisKeyInfo { @@ -68,6 +68,34 @@ test("collectExpandedGroupIds and flattenVisibleRedisKeyTree expand all search p ); }); +test("flattenVisibleRedisKeyTree handles very large expanded groups without stack overflow", () => { + const keys = Array.from({ length: 150_000 }, (_, index) => { + const id = String(index).padStart(6, "0"); + return makeKey(`user:${id}`, `user:${id}`); + }); + const tree = buildRedisKeyTree(keys, 0); + const expanded = collectExpandedGroupIds(tree); + + const rows = flattenVisibleRedisKeyTree(tree, expanded); + + assert.equal(rows.length, keys.length + 1); + assert.equal(rows[0]?.node.kind, "group"); + assert.equal(rows[0]?.node.label, "user"); + assert.equal(rows[1]?.depth, 1); + assert.equal(rows.at(-1)?.depth, 1); +}); + +test("redisKeyToFlatTreeRow keeps search results flat with the full key label", () => { + const row = redisKeyToFlatTreeRow(makeKey("user:profile:1", "user:profile:1", ""), 0); + + assert.equal(row.depth, 0); + assert.equal(row.node.kind, "leaf"); + if (row.node.kind !== "leaf") return; + assert.equal(row.node.label, "user:profile:1"); + assert.deepEqual(row.node.pathSegments, ["user:profile:1"]); + assert.equal(row.node.keyType, ""); +}); + test("collectRedisGroupKeyRaws returns every leaf key under a group", () => { const tree = buildRedisKeyTree([makeKey("user:profile:name", "k1"), makeKey("user:profile:email", "k2"), makeKey("user:settings", "k3"), makeKey("session:1", "k4")], 0); diff --git a/scripts/bench/redis-key-search.mjs b/scripts/bench/redis-key-search.mjs new file mode 100644 index 000000000..796df9ed3 --- /dev/null +++ b/scripts/bench/redis-key-search.mjs @@ -0,0 +1,384 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; +import { performance } from "node:perf_hooks"; + +const DEFAULTS = { + apiBase: "http://127.0.0.1:4224/api", + container: "dbx_bench_redis_key_search", + image: "redis:7-alpine", + host: "127.0.0.1", + port: 16151, + db: 0, + keyCount: 1_000_000, + keyPrefix: "user:", + pattern: "user:*", + scanCount: 10_000, + maxIterations: 15, + includeTyped: true, + json: false, +}; + +function parseArgs(argv) { + const options = { ...DEFAULTS }; + for (const arg of argv) { + if (arg === "--") continue; + const [rawKey, rawValue] = arg.split("=", 2); + const key = rawKey.replace(/^--/, ""); + const value = rawValue ?? "true"; + switch (key) { + case "api-base": + options.apiBase = value; + break; + case "container": + options.container = value; + break; + case "image": + options.image = value; + break; + case "port": + options.port = Number.parseInt(value, 10); + break; + case "key-count": + options.keyCount = Number.parseInt(value, 10); + break; + case "key-prefix": + options.keyPrefix = value; + break; + case "pattern": + options.pattern = value; + break; + case "scan-count": + options.scanCount = Number.parseInt(value, 10); + break; + case "max-iterations": + options.maxIterations = Number.parseInt(value, 10); + break; + case "typed": + options.includeTyped = value !== "false"; + break; + case "json": + options.json = value !== "false"; + break; + case "help": + printHelp(); + process.exit(0); + default: + throw new Error(`Unknown option: ${rawKey}`); + } + } + return options; +} + +function printHelp() { + console.log(`Redis key search benchmark + +Usage: + pnpm bench:redis-key-search [options] + +Options: + --api-base=http://127.0.0.1:4224/api DBX Web API base URL + --container=dbx_bench_redis_key_search Redis Docker container name + --port=16151 Host port for Redis + --key-count=1000000 Number of generated keys + --pattern=user:* SCAN MATCH pattern + --scan-count=10000 DBX SCAN COUNT value + --max-iterations=15 DBX server-side SCAN iterations per API call + --typed=false Skip DBX includeTypes=true comparison + --json Print JSON only + +Before running this benchmark, start DBX Web separately, for example: + DBX_DATA_DIR=/tmp/dbx-bench DBX_PORT=4224 DBX_DISABLE_PASSWORD=1 cargo run -p dbx-web +`); +} + +function assertFinitePositiveInteger(name, value) { + if (!Number.isFinite(value) || value <= 0 || Math.trunc(value) !== value) { + throw new Error(`${name} must be a positive integer, got ${value}`); + } +} + +function run(command, args, options = {}) { + const startedAt = performance.now(); + const child = spawn(command, args, { + cwd: process.cwd(), + env: process.env, + stdio: options.stdio ?? ["ignore", "pipe", "pipe"], + }); + const stdout = []; + const stderr = []; + if (child.stdout) child.stdout.on("data", (chunk) => stdout.push(chunk)); + if (child.stderr) child.stderr.on("data", (chunk) => stderr.push(chunk)); + return new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", (code) => { + const result = { + code, + stdout: Buffer.concat(stdout).toString(), + stderr: Buffer.concat(stderr).toString(), + elapsedMs: performance.now() - startedAt, + }; + if (code === 0) resolve(result); + else reject(new Error(`${command} ${args.join(" ")} failed with ${code}\n${result.stderr || result.stdout}`)); + }); + }); +} + +async function dockerExec(container, args) { + return run("docker", ["exec", container, ...args]); +} + +async function ensureRedisContainer(options) { + try { + await run("docker", ["inspect", options.container]); + await run("docker", ["start", options.container]); + } catch { + await run("docker", [ + "run", + "-d", + "--name", + options.container, + "-p", + `${options.host}:${options.port}:6379`, + options.image, + "redis-server", + "--save", + "", + "--appendonly", + "no", + ]); + } + + const deadline = Date.now() + 15_000; + while (Date.now() < deadline) { + try { + const pong = (await dockerExec(options.container, ["redis-cli", "PING"])).stdout.trim(); + if (pong === "PONG") return; + } catch { + await new Promise((resolve) => setTimeout(resolve, 250)); + } + } + throw new Error("Timed out waiting for Redis container"); +} + +async function redisDbSize(container) { + const out = await dockerExec(container, ["redis-cli", "DBSIZE"]); + return Number.parseInt(out.stdout.trim(), 10); +} + +function redisSetCommand(key, value = "1") { + const keyBytes = Buffer.byteLength(key); + const valueBytes = Buffer.byteLength(value); + return `*3\r\n$3\r\nSET\r\n$${keyBytes}\r\n${key}\r\n$${valueBytes}\r\n${value}\r\n`; +} + +async function seedRedis(options) { + const currentSize = await redisDbSize(options.container); + if (currentSize === options.keyCount) return { skipped: true, elapsedMs: 0 }; + + await dockerExec(options.container, ["redis-cli", "FLUSHALL"]); + + const startedAt = performance.now(); + const child = spawn("docker", ["exec", "-i", options.container, "redis-cli", "--pipe"], { + stdio: ["pipe", "pipe", "pipe"], + }); + const stderr = []; + child.stderr.on("data", (chunk) => stderr.push(chunk)); + + const width = Math.max(7, String(options.keyCount - 1).length); + for (let index = 0; index < options.keyCount; index += 1) { + const key = `${options.keyPrefix}${String(index).padStart(width, "0")}`; + if (!child.stdin.write(redisSetCommand(key))) { + await new Promise((resolve) => child.stdin.once("drain", resolve)); + } + } + child.stdin.end(); + + await new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) resolve(); + else reject(new Error(`redis-cli --pipe failed with ${code}\n${Buffer.concat(stderr).toString()}`)); + }); + }); + + return { skipped: false, elapsedMs: performance.now() - startedAt }; +} + +async function measureRedisCliScan(options) { + const startedAt = performance.now(); + const child = spawn("docker", ["exec", options.container, "redis-cli", "--scan", "--pattern", options.pattern], { + stdio: ["ignore", "pipe", "pipe"], + }); + let count = 0; + let pending = ""; + const stderr = []; + + child.stdout.on("data", (chunk) => { + pending += chunk.toString(); + const lines = pending.split(/\r?\n/); + pending = lines.pop() ?? ""; + count += lines.filter(Boolean).length; + }); + child.stderr.on("data", (chunk) => stderr.push(chunk)); + + await new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", (code) => { + if (pending.trim()) count += 1; + if (code === 0) resolve(); + else reject(new Error(`redis-cli --scan failed with ${code}\n${Buffer.concat(stderr).toString()}`)); + }); + }); + + return { label: "redis-cli --scan", keys: count, elapsedMs: performance.now() - startedAt }; +} + +async function postJsonText(url, body) { + const response = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const text = await response.text(); + if (!response.ok) throw new Error(`${url} ${response.status}\n${text}`); + return { text, json: JSON.parse(text) }; +} + +async function ensureDbxConnection(options) { + const connectionId = `bench-redis-key-search-${options.port}`; + const config = { + id: connectionId, + name: `Bench Redis Key Search ${options.port}`, + db_type: "redis", + host: options.host, + port: options.port, + username: "", + password: "", + database: String(options.db), + auth_database: "", + ssl: false, + redis_key_separator: ":", + connection_timeout: 10, + connect_timeout_secs: 10, + }; + + await postJsonText(`${options.apiBase}/connection/save`, { configs: [config] }); + await postJsonText(`${options.apiBase}/connection/connect`, { config }); + return connectionId; +} + +async function measureDbxScan(options, connectionId, includeTypes) { + let cursor = 0; + let calls = 0; + let keys = 0; + let payloadBytes = 0; + const startedAt = performance.now(); + + do { + const { text, json } = await postJsonText(`${options.apiBase}/redis/scan-keys-batch`, { + connectionId, + db: options.db, + cursor, + pattern: options.pattern, + count: options.scanCount, + maxIterations: options.maxIterations, + includeTypes, + }); + payloadBytes += Buffer.byteLength(text); + cursor = json.cursor; + keys += json.keys.length; + calls += 1; + } while (cursor !== 0); + + return { + label: includeTypes ? "DBX scan-keys-batch includeTypes=true" : "DBX scan-keys-batch includeTypes=false", + keys, + calls, + payloadBytes, + elapsedMs: performance.now() - startedAt, + }; +} + +function formatMs(ms) { + return `${Math.round(ms)}ms`; +} + +function formatBytes(bytes) { + if (!bytes) return "0 B"; + const units = ["B", "KB", "MB", "GB"]; + let value = bytes; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit += 1; + } + return `${value.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`; +} + +function printReport(result) { + console.log("# Redis key search benchmark"); + console.log(""); + console.log(`- API: ${result.config.apiBase}`); + console.log(`- Redis: ${result.config.container} on ${result.config.host}:${result.config.port}`); + console.log(`- Keys: ${result.config.keyCount}`); + console.log(`- Pattern: ${result.config.pattern}`); + console.log(`- DBX scan count: ${result.config.scanCount}`); + console.log(`- DBX max iterations: ${result.config.maxIterations}`); + console.log(`- Seed: ${result.seed.skipped ? "reused existing dataset" : `loaded in ${formatMs(result.seed.elapsedMs)}`}`); + console.log(""); + console.log("| Case | Keys | Calls | Payload | Time |"); + console.log("| --- | ---: | ---: | ---: | ---: |"); + for (const row of result.measurements) { + console.log(`| ${row.label} | ${row.keys} | ${row.calls ?? "-"} | ${row.payloadBytes ? formatBytes(row.payloadBytes) : "-"} | ${formatMs(row.elapsedMs)} |`); + } +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + assertFinitePositiveInteger("port", options.port); + assertFinitePositiveInteger("key-count", options.keyCount); + assertFinitePositiveInteger("scan-count", options.scanCount); + assertFinitePositiveInteger("max-iterations", options.maxIterations); + + if (!options.json) { + console.log("Preparing Redis benchmark dataset..."); + } + await ensureRedisContainer(options); + const seed = await seedRedis(options); + + if (!options.json) { + console.log("Connecting DBX Web API..."); + } + const connectionId = await ensureDbxConnection(options); + + const measurements = []; + measurements.push(await measureRedisCliScan(options)); + measurements.push(await measureDbxScan(options, connectionId, false)); + if (options.includeTyped) { + measurements.push(await measureDbxScan(options, connectionId, true)); + } + + const result = { + config: { + apiBase: options.apiBase, + container: options.container, + host: options.host, + port: options.port, + keyCount: options.keyCount, + pattern: options.pattern, + scanCount: options.scanCount, + maxIterations: options.maxIterations, + }, + seed, + measurements, + generatedAt: new Date().toISOString(), + }; + + if (options.json) console.log(JSON.stringify(result, null, 2)); + else printReport(result); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)); + process.exit(1); +});