perf(redis): reduce large key search payload and render cost (#1672)
This commit is contained in:
Guoyu Su
GitHub
parent
f29d393c37
commit
90fd77cb9e
|
|
@ -22,7 +22,7 @@ import type { RedisKeyInfo, RedisScanResult, RedisValue, HistoryEntry } from "@/
|
|||
import { uuid } from "@/lib/utils";
|
||||
import { useConnectionStore } from "@/stores/connectionStore";
|
||||
import { useSettingsStore } from "@/stores/settingsStore";
|
||||
import { buildRedisKeyTree, collectExpandedGroupIds, collectRedisGroupKeyRaws, flattenVisibleRedisKeyTree, mergeKeysIntoRedisKeyTree, type RedisKeyTreeNode } from "@/lib/redisKeyTree";
|
||||
import { buildRedisKeyTree, collectExpandedGroupIds, collectRedisGroupKeyRaws, flattenVisibleRedisKeyTree, mergeKeysIntoRedisKeyTree, redisKeyToFlatTreeRow, type RedisKeyTreeNode } from "@/lib/redisKeyTree";
|
||||
import { classifyRedisCommandSafety } from "@/lib/redisCommandSafety";
|
||||
import { isRedisMutatingCommand } from "@/lib/redisCommandTable";
|
||||
import { isRedisClearScreenCommand, nextRedisCommandDb, redisKeyTextToRaw } from "@/lib/redisCommandSession";
|
||||
|
|
@ -108,6 +108,7 @@ const valueQuery = computed(() => searchPattern.value.trim());
|
|||
const isValueSearchMode = computed(() => searchMode.value === "value" || searchMode.value === "all");
|
||||
const effectivePattern = computed(() => (searchMode.value === "key" ? redisKeySearchPattern(searchPattern.value, fuzzyKeySearch.value) : "*"));
|
||||
const isSearchMode = computed(() => (searchMode.value === "key" ? effectivePattern.value !== "*" : valueQuery.value !== ""));
|
||||
const useFlatKeySearchRows = computed(() => searchMode.value === "key" && isSearchMode.value);
|
||||
const searchPlaceholder = computed(() => {
|
||||
if (searchMode.value === "key") return fuzzyKeySearch.value ? t("redis.fuzzyPattern") : t("redis.pattern");
|
||||
return searchMode.value === "all" ? t("redis.allSearchPlaceholder") : t("redis.valueSearchPlaceholder");
|
||||
|
|
@ -150,12 +151,13 @@ const createKeyTypeOptions = computed<{ value: RedisCreateKeyType; label: string
|
|||
{ value: "stream", label: "Stream" },
|
||||
{ value: "json", label: "JSON" },
|
||||
]);
|
||||
const visibleRows = computed(() =>
|
||||
flattenVisibleRedisKeyTree(treeKeys.value, expandedGroupIds.value).map((row) => ({
|
||||
const visibleRows = computed(() => {
|
||||
const rows = useFlatKeySearchRows.value ? flatKeys.value.map((key) => redisKeyToFlatTreeRow(key, props.db)) : flattenVisibleRedisKeyTree(treeKeys.value, expandedGroupIds.value);
|
||||
return rows.map((row) => ({
|
||||
...row,
|
||||
id: row.node.id,
|
||||
})),
|
||||
);
|
||||
}));
|
||||
});
|
||||
let commandHistoryId = 0;
|
||||
|
||||
function countLeaves(node: RedisKeyTreeNode): number {
|
||||
|
|
@ -236,7 +238,10 @@ function appendScanResult(result: RedisScanResult, options: { updateTree?: boole
|
|||
}
|
||||
|
||||
if (options.updateTree ?? true) {
|
||||
if (treeKeys.value.length === 0) {
|
||||
if (useFlatKeySearchRows.value) {
|
||||
treeKeys.value = [];
|
||||
expandedGroupIds.value = new Set();
|
||||
} else if (treeKeys.value.length === 0) {
|
||||
rebuildTree(isSearchMode.value);
|
||||
} else {
|
||||
mergeTree(newKeys);
|
||||
|
|
@ -345,7 +350,7 @@ async function fetchAll() {
|
|||
}
|
||||
} finally {
|
||||
if (requestId === searchRequestId) {
|
||||
if (changed) rebuildTree(isSearchMode.value);
|
||||
if (changed && !useFlatKeySearchRows.value) rebuildTree(isSearchMode.value);
|
||||
isFetchingAll.value = false;
|
||||
}
|
||||
}
|
||||
|
|
@ -1012,7 +1017,7 @@ defineExpose({ focusSearch });
|
|||
</div>
|
||||
|
||||
<div class="flex shrink-0 items-center justify-end gap-1">
|
||||
<Badge v-if="row.node.kind === 'leaf' && row.node.keyType !== 'unknown'" variant="outline" class="text-xs px-1.5 py-0" :class="typeColor(row.node.keyType)">{{ row.node.keyType }}</Badge>
|
||||
<Badge v-if="row.node.kind === 'leaf' && row.node.keyType" variant="outline" class="text-xs px-1.5 py-0" :class="typeColor(row.node.keyType)">{{ row.node.keyType }}</Badge>
|
||||
<Button v-if="row.node.kind === 'group'" variant="ghost" size="icon" class="h-5 w-5 shrink-0 text-destructive opacity-0 group-hover:opacity-100" :title="t('redis.deleteGroup')" @click="requestGroupDelete(row.node, $event)">
|
||||
<Trash2 class="h-3 w-3" />
|
||||
</Button>
|
||||
|
|
|
|||
|
|
@ -182,12 +182,13 @@ const isBinaryStringValue = computed(() => data.value?.key_type === "string" &&
|
|||
const hasMore = computed(() => scanCursor.value != null && scanCursor.value > 0);
|
||||
const metadataSizeLabel = computed(() => {
|
||||
const metadata = props.metadata;
|
||||
if (!metadata || metadata.size <= 0) return "";
|
||||
const size = metadata?.size ?? 0;
|
||||
if (!metadata || size <= 0) return "";
|
||||
if (metadata.key_type === "string") {
|
||||
if (metadata.size >= 1024) return `${(metadata.size / 1024).toFixed(1)} KB`;
|
||||
return `${metadata.size} B`;
|
||||
if (size >= 1024) return `${(size / 1024).toFixed(1)} KB`;
|
||||
return `${size} B`;
|
||||
}
|
||||
return String(metadata.size);
|
||||
return String(size);
|
||||
});
|
||||
|
||||
function collectionCountLabel(kind: "items" | "fields" | "members", loaded: number, total?: number | null) {
|
||||
|
|
|
|||
|
|
@ -37,6 +37,25 @@ function buildLeafId(db: number, keyRaw: string): string {
|
|||
return `leaf:${db}:${keyRaw}`;
|
||||
}
|
||||
|
||||
export function redisKeyToFlatTreeRow(key: RedisKeyInfo, db: number): RedisKeyTreeRow {
|
||||
return {
|
||||
node: {
|
||||
kind: "leaf",
|
||||
id: buildLeafId(db, key.key_raw),
|
||||
label: key.key_display,
|
||||
fullKeyDisplay: key.key_display,
|
||||
keyRaw: key.key_raw,
|
||||
db,
|
||||
keyType: key.key_type ?? "",
|
||||
ttl: key.ttl ?? -2,
|
||||
size: key.size ?? 0,
|
||||
valuePreview: key.value_preview ?? "",
|
||||
pathSegments: [key.key_display],
|
||||
},
|
||||
depth: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function compareRedisTreeNodes(a: RedisKeyTreeNode, b: RedisKeyTreeNode): number {
|
||||
if (a.kind !== b.kind) return a.kind === "group" ? -1 : 1;
|
||||
return a.label.localeCompare(b.label);
|
||||
|
|
@ -74,10 +93,10 @@ function insertKeyIntoTree(root: RedisKeyTreeNode[], groupMap: Map<string, Redis
|
|||
fullKeyDisplay: key.key_display,
|
||||
keyRaw: key.key_raw,
|
||||
db,
|
||||
keyType: key.key_type,
|
||||
ttl: key.ttl,
|
||||
size: key.size,
|
||||
valuePreview: key.value_preview,
|
||||
keyType: key.key_type ?? "",
|
||||
ttl: key.ttl ?? -2,
|
||||
size: key.size ?? 0,
|
||||
valuePreview: key.value_preview ?? "",
|
||||
pathSegments,
|
||||
});
|
||||
return;
|
||||
|
|
@ -110,10 +129,10 @@ function insertKeyIntoTree(root: RedisKeyTreeNode[], groupMap: Map<string, Redis
|
|||
fullKeyDisplay: key.key_display,
|
||||
keyRaw: key.key_raw,
|
||||
db,
|
||||
keyType: key.key_type,
|
||||
ttl: key.ttl,
|
||||
size: key.size,
|
||||
valuePreview: key.value_preview,
|
||||
keyType: key.key_type ?? "",
|
||||
ttl: key.ttl ?? -2,
|
||||
size: key.size ?? 0,
|
||||
valuePreview: key.value_preview ?? "",
|
||||
pathSegments,
|
||||
});
|
||||
}
|
||||
|
|
@ -192,11 +211,21 @@ export function collectRedisGroupKeyRaws(group: RedisKeyTreeGroupNode): string[]
|
|||
|
||||
export function flattenVisibleRedisKeyTree(nodes: RedisKeyTreeNode[], expandedGroupIds: ReadonlySet<string>, depth = 0): RedisKeyTreeRow[] {
|
||||
const rows: RedisKeyTreeRow[] = [];
|
||||
const stack: RedisKeyTreeRow[] = [];
|
||||
|
||||
for (const node of nodes) {
|
||||
rows.push({ node, depth });
|
||||
if (node.kind === "group" && expandedGroupIds.has(node.id)) {
|
||||
rows.push(...flattenVisibleRedisKeyTree(node.children, expandedGroupIds, depth + 1));
|
||||
for (let index = nodes.length - 1; index >= 0; index--) {
|
||||
stack.push({ node: nodes[index], depth });
|
||||
}
|
||||
|
||||
while (stack.length > 0) {
|
||||
const row = stack.pop()!;
|
||||
rows.push(row);
|
||||
|
||||
if (row.node.kind !== "group" || !expandedGroupIds.has(row.node.id)) continue;
|
||||
|
||||
const childDepth = row.depth + 1;
|
||||
for (let index = row.node.children.length - 1; index >= 0; index--) {
|
||||
stack.push({ node: row.node.children[index], depth: childDepth });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1113,10 +1113,10 @@ export async function getAppVersion(): Promise<string> {
|
|||
export interface RedisKeyInfo {
|
||||
key_display: string;
|
||||
key_raw: string;
|
||||
key_type: string;
|
||||
ttl: number;
|
||||
size: number;
|
||||
value_preview: string;
|
||||
key_type?: string;
|
||||
ttl?: number;
|
||||
size?: number;
|
||||
value_preview?: string;
|
||||
}
|
||||
|
||||
export interface RedisDatabaseInfo {
|
||||
|
|
|
|||
|
|
@ -30,12 +30,28 @@ pub struct RedisDatabaseInfo {
|
|||
pub struct RedisKeyInfo {
|
||||
pub key_display: String,
|
||||
pub key_raw: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub key_type: String,
|
||||
#[serde(default = "default_missing_ttl", skip_serializing_if = "is_missing_ttl")]
|
||||
pub ttl: i64,
|
||||
#[serde(default, skip_serializing_if = "is_zero_u64")]
|
||||
pub size: u64,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub value_preview: String,
|
||||
}
|
||||
|
||||
fn default_missing_ttl() -> i64 {
|
||||
-2
|
||||
}
|
||||
|
||||
fn is_missing_ttl(ttl: &i64) -> bool {
|
||||
*ttl == -2
|
||||
}
|
||||
|
||||
fn is_zero_u64(value: &u64) -> bool {
|
||||
*value == 0
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RedisScanResult {
|
||||
pub cursor: u64,
|
||||
|
|
@ -1510,7 +1526,7 @@ where
|
|||
let key_type = if include_types {
|
||||
key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string())
|
||||
} else {
|
||||
"unknown".to_string()
|
||||
String::new()
|
||||
};
|
||||
let value_preview = if include_types {
|
||||
redis_key_value_preview(key_types.get(index).map(String::as_str).unwrap_or("unknown"))
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
"build": "vite build --config apps/desktop/vite.config.ts",
|
||||
"build:checked": "pnpm typecheck && pnpm build",
|
||||
"check": "node scripts/run-check.mjs",
|
||||
"bench:redis-key-search": "node scripts/bench/redis-key-search.mjs",
|
||||
"lint": "oxlint --vue-plugin apps/desktop/src",
|
||||
"fmt": "oxfmt \"apps/desktop/src/**/*.{ts,vue}\"",
|
||||
"test": "vitest run",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { test } from "vitest";
|
||||
import assert from "node:assert/strict";
|
||||
import { buildRedisKeyTree, collectRedisGroupKeyRaws, collectExpandedGroupIds, flattenVisibleRedisKeyTree, mergeKeysIntoRedisKeyTree, type RedisKeyTreeNode } from "../../apps/desktop/src/lib/redisKeyTree.ts";
|
||||
import { buildRedisKeyTree, collectRedisGroupKeyRaws, collectExpandedGroupIds, flattenVisibleRedisKeyTree, mergeKeysIntoRedisKeyTree, redisKeyToFlatTreeRow, type RedisKeyTreeNode } from "../../apps/desktop/src/lib/redisKeyTree.ts";
|
||||
import type { RedisKeyInfo } from "../../apps/desktop/src/lib/api.ts";
|
||||
|
||||
function makeKey(key_display: string, key_raw: string, key_type = "string", ttl = -1): RedisKeyInfo {
|
||||
|
|
@ -68,6 +68,34 @@ test("collectExpandedGroupIds and flattenVisibleRedisKeyTree expand all search p
|
|||
);
|
||||
});
|
||||
|
||||
test("flattenVisibleRedisKeyTree handles very large expanded groups without stack overflow", () => {
|
||||
const keys = Array.from({ length: 150_000 }, (_, index) => {
|
||||
const id = String(index).padStart(6, "0");
|
||||
return makeKey(`user:${id}`, `user:${id}`);
|
||||
});
|
||||
const tree = buildRedisKeyTree(keys, 0);
|
||||
const expanded = collectExpandedGroupIds(tree);
|
||||
|
||||
const rows = flattenVisibleRedisKeyTree(tree, expanded);
|
||||
|
||||
assert.equal(rows.length, keys.length + 1);
|
||||
assert.equal(rows[0]?.node.kind, "group");
|
||||
assert.equal(rows[0]?.node.label, "user");
|
||||
assert.equal(rows[1]?.depth, 1);
|
||||
assert.equal(rows.at(-1)?.depth, 1);
|
||||
});
|
||||
|
||||
test("redisKeyToFlatTreeRow keeps search results flat with the full key label", () => {
|
||||
const row = redisKeyToFlatTreeRow(makeKey("user:profile:1", "user:profile:1", ""), 0);
|
||||
|
||||
assert.equal(row.depth, 0);
|
||||
assert.equal(row.node.kind, "leaf");
|
||||
if (row.node.kind !== "leaf") return;
|
||||
assert.equal(row.node.label, "user:profile:1");
|
||||
assert.deepEqual(row.node.pathSegments, ["user:profile:1"]);
|
||||
assert.equal(row.node.keyType, "");
|
||||
});
|
||||
|
||||
test("collectRedisGroupKeyRaws returns every leaf key under a group", () => {
|
||||
const tree = buildRedisKeyTree([makeKey("user:profile:name", "k1"), makeKey("user:profile:email", "k2"), makeKey("user:settings", "k3"), makeKey("session:1", "k4")], 0);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,384 @@
|
|||
#!/usr/bin/env node
|
||||
import { spawn } from "node:child_process";
|
||||
import { performance } from "node:perf_hooks";
|
||||
|
||||
const DEFAULTS = {
|
||||
apiBase: "http://127.0.0.1:4224/api",
|
||||
container: "dbx_bench_redis_key_search",
|
||||
image: "redis:7-alpine",
|
||||
host: "127.0.0.1",
|
||||
port: 16151,
|
||||
db: 0,
|
||||
keyCount: 1_000_000,
|
||||
keyPrefix: "user:",
|
||||
pattern: "user:*",
|
||||
scanCount: 10_000,
|
||||
maxIterations: 15,
|
||||
includeTyped: true,
|
||||
json: false,
|
||||
};
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = { ...DEFAULTS };
|
||||
for (const arg of argv) {
|
||||
if (arg === "--") continue;
|
||||
const [rawKey, rawValue] = arg.split("=", 2);
|
||||
const key = rawKey.replace(/^--/, "");
|
||||
const value = rawValue ?? "true";
|
||||
switch (key) {
|
||||
case "api-base":
|
||||
options.apiBase = value;
|
||||
break;
|
||||
case "container":
|
||||
options.container = value;
|
||||
break;
|
||||
case "image":
|
||||
options.image = value;
|
||||
break;
|
||||
case "port":
|
||||
options.port = Number.parseInt(value, 10);
|
||||
break;
|
||||
case "key-count":
|
||||
options.keyCount = Number.parseInt(value, 10);
|
||||
break;
|
||||
case "key-prefix":
|
||||
options.keyPrefix = value;
|
||||
break;
|
||||
case "pattern":
|
||||
options.pattern = value;
|
||||
break;
|
||||
case "scan-count":
|
||||
options.scanCount = Number.parseInt(value, 10);
|
||||
break;
|
||||
case "max-iterations":
|
||||
options.maxIterations = Number.parseInt(value, 10);
|
||||
break;
|
||||
case "typed":
|
||||
options.includeTyped = value !== "false";
|
||||
break;
|
||||
case "json":
|
||||
options.json = value !== "false";
|
||||
break;
|
||||
case "help":
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
default:
|
||||
throw new Error(`Unknown option: ${rawKey}`);
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Redis key search benchmark
|
||||
|
||||
Usage:
|
||||
pnpm bench:redis-key-search [options]
|
||||
|
||||
Options:
|
||||
--api-base=http://127.0.0.1:4224/api DBX Web API base URL
|
||||
--container=dbx_bench_redis_key_search Redis Docker container name
|
||||
--port=16151 Host port for Redis
|
||||
--key-count=1000000 Number of generated keys
|
||||
--pattern=user:* SCAN MATCH pattern
|
||||
--scan-count=10000 DBX SCAN COUNT value
|
||||
--max-iterations=15 DBX server-side SCAN iterations per API call
|
||||
--typed=false Skip DBX includeTypes=true comparison
|
||||
--json Print JSON only
|
||||
|
||||
Before running this benchmark, start DBX Web separately, for example:
|
||||
DBX_DATA_DIR=/tmp/dbx-bench DBX_PORT=4224 DBX_DISABLE_PASSWORD=1 cargo run -p dbx-web
|
||||
`);
|
||||
}
|
||||
|
||||
function assertFinitePositiveInteger(name, value) {
|
||||
if (!Number.isFinite(value) || value <= 0 || Math.trunc(value) !== value) {
|
||||
throw new Error(`${name} must be a positive integer, got ${value}`);
|
||||
}
|
||||
}
|
||||
|
||||
function run(command, args, options = {}) {
|
||||
const startedAt = performance.now();
|
||||
const child = spawn(command, args, {
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
stdio: options.stdio ?? ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = [];
|
||||
const stderr = [];
|
||||
if (child.stdout) child.stdout.on("data", (chunk) => stdout.push(chunk));
|
||||
if (child.stderr) child.stderr.on("data", (chunk) => stderr.push(chunk));
|
||||
return new Promise((resolve, reject) => {
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
const result = {
|
||||
code,
|
||||
stdout: Buffer.concat(stdout).toString(),
|
||||
stderr: Buffer.concat(stderr).toString(),
|
||||
elapsedMs: performance.now() - startedAt,
|
||||
};
|
||||
if (code === 0) resolve(result);
|
||||
else reject(new Error(`${command} ${args.join(" ")} failed with ${code}\n${result.stderr || result.stdout}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function dockerExec(container, args) {
|
||||
return run("docker", ["exec", container, ...args]);
|
||||
}
|
||||
|
||||
async function ensureRedisContainer(options) {
|
||||
try {
|
||||
await run("docker", ["inspect", options.container]);
|
||||
await run("docker", ["start", options.container]);
|
||||
} catch {
|
||||
await run("docker", [
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
options.container,
|
||||
"-p",
|
||||
`${options.host}:${options.port}:6379`,
|
||||
options.image,
|
||||
"redis-server",
|
||||
"--save",
|
||||
"",
|
||||
"--appendonly",
|
||||
"no",
|
||||
]);
|
||||
}
|
||||
|
||||
const deadline = Date.now() + 15_000;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const pong = (await dockerExec(options.container, ["redis-cli", "PING"])).stdout.trim();
|
||||
if (pong === "PONG") return;
|
||||
} catch {
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
}
|
||||
throw new Error("Timed out waiting for Redis container");
|
||||
}
|
||||
|
||||
async function redisDbSize(container) {
|
||||
const out = await dockerExec(container, ["redis-cli", "DBSIZE"]);
|
||||
return Number.parseInt(out.stdout.trim(), 10);
|
||||
}
|
||||
|
||||
function redisSetCommand(key, value = "1") {
|
||||
const keyBytes = Buffer.byteLength(key);
|
||||
const valueBytes = Buffer.byteLength(value);
|
||||
return `*3\r\n$3\r\nSET\r\n$${keyBytes}\r\n${key}\r\n$${valueBytes}\r\n${value}\r\n`;
|
||||
}
|
||||
|
||||
async function seedRedis(options) {
|
||||
const currentSize = await redisDbSize(options.container);
|
||||
if (currentSize === options.keyCount) return { skipped: true, elapsedMs: 0 };
|
||||
|
||||
await dockerExec(options.container, ["redis-cli", "FLUSHALL"]);
|
||||
|
||||
const startedAt = performance.now();
|
||||
const child = spawn("docker", ["exec", "-i", options.container, "redis-cli", "--pipe"], {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
const stderr = [];
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
||||
|
||||
const width = Math.max(7, String(options.keyCount - 1).length);
|
||||
for (let index = 0; index < options.keyCount; index += 1) {
|
||||
const key = `${options.keyPrefix}${String(index).padStart(width, "0")}`;
|
||||
if (!child.stdin.write(redisSetCommand(key))) {
|
||||
await new Promise((resolve) => child.stdin.once("drain", resolve));
|
||||
}
|
||||
}
|
||||
child.stdin.end();
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`redis-cli --pipe failed with ${code}\n${Buffer.concat(stderr).toString()}`));
|
||||
});
|
||||
});
|
||||
|
||||
return { skipped: false, elapsedMs: performance.now() - startedAt };
|
||||
}
|
||||
|
||||
async function measureRedisCliScan(options) {
|
||||
const startedAt = performance.now();
|
||||
const child = spawn("docker", ["exec", options.container, "redis-cli", "--scan", "--pattern", options.pattern], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let count = 0;
|
||||
let pending = "";
|
||||
const stderr = [];
|
||||
|
||||
child.stdout.on("data", (chunk) => {
|
||||
pending += chunk.toString();
|
||||
const lines = pending.split(/\r?\n/);
|
||||
pending = lines.pop() ?? "";
|
||||
count += lines.filter(Boolean).length;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (pending.trim()) count += 1;
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`redis-cli --scan failed with ${code}\n${Buffer.concat(stderr).toString()}`));
|
||||
});
|
||||
});
|
||||
|
||||
return { label: "redis-cli --scan", keys: count, elapsedMs: performance.now() - startedAt };
|
||||
}
|
||||
|
||||
async function postJsonText(url, body) {
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
const text = await response.text();
|
||||
if (!response.ok) throw new Error(`${url} ${response.status}\n${text}`);
|
||||
return { text, json: JSON.parse(text) };
|
||||
}
|
||||
|
||||
async function ensureDbxConnection(options) {
|
||||
const connectionId = `bench-redis-key-search-${options.port}`;
|
||||
const config = {
|
||||
id: connectionId,
|
||||
name: `Bench Redis Key Search ${options.port}`,
|
||||
db_type: "redis",
|
||||
host: options.host,
|
||||
port: options.port,
|
||||
username: "",
|
||||
password: "",
|
||||
database: String(options.db),
|
||||
auth_database: "",
|
||||
ssl: false,
|
||||
redis_key_separator: ":",
|
||||
connection_timeout: 10,
|
||||
connect_timeout_secs: 10,
|
||||
};
|
||||
|
||||
await postJsonText(`${options.apiBase}/connection/save`, { configs: [config] });
|
||||
await postJsonText(`${options.apiBase}/connection/connect`, { config });
|
||||
return connectionId;
|
||||
}
|
||||
|
||||
async function measureDbxScan(options, connectionId, includeTypes) {
|
||||
let cursor = 0;
|
||||
let calls = 0;
|
||||
let keys = 0;
|
||||
let payloadBytes = 0;
|
||||
const startedAt = performance.now();
|
||||
|
||||
do {
|
||||
const { text, json } = await postJsonText(`${options.apiBase}/redis/scan-keys-batch`, {
|
||||
connectionId,
|
||||
db: options.db,
|
||||
cursor,
|
||||
pattern: options.pattern,
|
||||
count: options.scanCount,
|
||||
maxIterations: options.maxIterations,
|
||||
includeTypes,
|
||||
});
|
||||
payloadBytes += Buffer.byteLength(text);
|
||||
cursor = json.cursor;
|
||||
keys += json.keys.length;
|
||||
calls += 1;
|
||||
} while (cursor !== 0);
|
||||
|
||||
return {
|
||||
label: includeTypes ? "DBX scan-keys-batch includeTypes=true" : "DBX scan-keys-batch includeTypes=false",
|
||||
keys,
|
||||
calls,
|
||||
payloadBytes,
|
||||
elapsedMs: performance.now() - startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function formatMs(ms) {
|
||||
return `${Math.round(ms)}ms`;
|
||||
}
|
||||
|
||||
function formatBytes(bytes) {
|
||||
if (!bytes) return "0 B";
|
||||
const units = ["B", "KB", "MB", "GB"];
|
||||
let value = bytes;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
return `${value.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
function printReport(result) {
|
||||
console.log("# Redis key search benchmark");
|
||||
console.log("");
|
||||
console.log(`- API: ${result.config.apiBase}`);
|
||||
console.log(`- Redis: ${result.config.container} on ${result.config.host}:${result.config.port}`);
|
||||
console.log(`- Keys: ${result.config.keyCount}`);
|
||||
console.log(`- Pattern: ${result.config.pattern}`);
|
||||
console.log(`- DBX scan count: ${result.config.scanCount}`);
|
||||
console.log(`- DBX max iterations: ${result.config.maxIterations}`);
|
||||
console.log(`- Seed: ${result.seed.skipped ? "reused existing dataset" : `loaded in ${formatMs(result.seed.elapsedMs)}`}`);
|
||||
console.log("");
|
||||
console.log("| Case | Keys | Calls | Payload | Time |");
|
||||
console.log("| --- | ---: | ---: | ---: | ---: |");
|
||||
for (const row of result.measurements) {
|
||||
console.log(`| ${row.label} | ${row.keys} | ${row.calls ?? "-"} | ${row.payloadBytes ? formatBytes(row.payloadBytes) : "-"} | ${formatMs(row.elapsedMs)} |`);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
assertFinitePositiveInteger("port", options.port);
|
||||
assertFinitePositiveInteger("key-count", options.keyCount);
|
||||
assertFinitePositiveInteger("scan-count", options.scanCount);
|
||||
assertFinitePositiveInteger("max-iterations", options.maxIterations);
|
||||
|
||||
if (!options.json) {
|
||||
console.log("Preparing Redis benchmark dataset...");
|
||||
}
|
||||
await ensureRedisContainer(options);
|
||||
const seed = await seedRedis(options);
|
||||
|
||||
if (!options.json) {
|
||||
console.log("Connecting DBX Web API...");
|
||||
}
|
||||
const connectionId = await ensureDbxConnection(options);
|
||||
|
||||
const measurements = [];
|
||||
measurements.push(await measureRedisCliScan(options));
|
||||
measurements.push(await measureDbxScan(options, connectionId, false));
|
||||
if (options.includeTyped) {
|
||||
measurements.push(await measureDbxScan(options, connectionId, true));
|
||||
}
|
||||
|
||||
const result = {
|
||||
config: {
|
||||
apiBase: options.apiBase,
|
||||
container: options.container,
|
||||
host: options.host,
|
||||
port: options.port,
|
||||
keyCount: options.keyCount,
|
||||
pattern: options.pattern,
|
||||
scanCount: options.scanCount,
|
||||
maxIterations: options.maxIterations,
|
||||
},
|
||||
seed,
|
||||
measurements,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
if (options.json) console.log(JSON.stringify(result, null, 2));
|
||||
else printReport(result);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack || error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
Loading…
Reference in New Issue