diff --git a/Cargo.lock b/Cargo.lock index 8bcd713b9..09243d4d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1907,6 +1907,7 @@ dependencies = [ "font-kit", "futures", "iana-time-zone", + "jsonwebtoken", "log", "mongodb", "mysql_async", @@ -4029,6 +4030,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +dependencies = [ + "base64 0.22.1", + "js-sys", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "keccak" version = "0.2.0" @@ -7385,6 +7401,18 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_asn1" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror 2.0.18", + "time", +] + [[package]] name = "siphasher" version = "1.0.3" diff --git a/apps/desktop/public/icons/database/pulsar.svg b/apps/desktop/public/icons/database/pulsar.svg new file mode 100644 index 000000000..feca270e5 --- /dev/null +++ b/apps/desktop/public/icons/database/pulsar.svg @@ -0,0 +1,3 @@ + + + diff --git a/apps/desktop/src/App.vue b/apps/desktop/src/App.vue index 8cc6323d9..10eb8c91c 100644 --- a/apps/desktop/src/App.vue +++ b/apps/desktop/src/App.vue @@ -31,6 +31,7 @@ import "@/i18n"; import { translateBackendError } from "@/i18n/backend-errors"; import * as api from "@/lib/api"; import { connectionRedactedNameLabel } from "@/lib/connectionPresentation"; +import { quickConnectionOpenTarget } from "@/lib/connectionOpenTarget"; import { resolveDefaultDatabase } from "@/lib/defaultDatabase"; import { findTreeNodeById, resolveNewQueryTarget } from "@/lib/newQueryContext"; import { buildExecutableObjectSourceStatements, objectSourceSaveExecutionMode } from "@/lib/objectSourceEditor"; @@ -729,11 +730,19 @@ async function openConnectionQuery(connectionId: string) { const connection = connectionStore.getConfig(connectionId); if (!connection) return; connectionStore.activeConnectionId = connectionId; - const tabId = queryStore.createTab(connectionId, resolveDefaultDatabase(connection, [])); + const initialTarget = quickConnectionOpenTarget(connection); + if (initialTarget.kind === "mq-admin") { + queryStore.openMqAdmin(connectionId); + return; + } + const tabId = queryStore.createTab(connectionId, initialTarget.database); try { await connectionStore.ensureConnected(connectionId); const options = await getDatabaseOptions(connectionId); - queryStore.updateDatabase(tabId, resolveDefaultDatabase(connection, options)); + const target = quickConnectionOpenTarget(connection, options); + if (target.kind === "query") { + queryStore.updateDatabase(tabId, target.database); + } } catch (e: any) { toast( t("connection.connectFailed", { diff --git a/apps/desktop/src/components/connection/ConnectionDialog.vue b/apps/desktop/src/components/connection/ConnectionDialog.vue index 1a9490248..180dbcd54 100644 --- a/apps/desktop/src/components/connection/ConnectionDialog.vue +++ b/apps/desktop/src/components/connection/ConnectionDialog.vue @@ -14,6 +14,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Switch } from "@/components/ui/switch"; import type { ConnectionConfig, DatabaseType, JdbcDriverInfo, JdbcMavenBundleInfo, ProxyTunnelConfig, SshTunnelConfig, TransportLayerConfig } from "@/types/database"; +import type { MqAdminConfig, MqAuth, MqSystemKind } from "@/types/mq"; import { useConnectionStore } from "@/stores/connectionStore"; import { useSettingsStore } from "@/stores/settingsStore"; import { useToast } from "@/composables/useToast"; @@ -25,6 +26,7 @@ import type { ConnectionDeepLinkDraft } from "@/lib/connectionDeepLink"; import { connectionUrlPlaceholder as getUrlPlaceholder } from "@/lib/connectionPresentation"; import { h2ConnectionModeForConfig, h2FileJdbcUrl, h2FilePathFromJdbcUrl, type H2ConnectionMode } from "@/lib/h2Connection"; import { isLocalFileTypeDb } from "@/lib/connectionFile"; +import { MQ_PINNED_VERSION_OPTIONS, pinnedVersionToSelection, selectionToPinnedVersion } from "@/lib/mqPinnedVersionOptions"; import { mongodbAuthFailureHint, mongoUrlParam, setMongoUrlParam } from "@/lib/mongoConnectionOptions"; import { copyToClipboard } from "@/lib/clipboard"; import { showAgentDriverInstallHint, type AgentDriverInstallState } from "@/lib/agentDriverInstallHint"; @@ -37,6 +39,7 @@ type DbCategory = { key: string; title: string; options: DbOption[] }; type DialogStep = "select" | "config"; type DbPickerView = "icon" | "list"; type ConfigTab = "connection" | "advanced" | "tls" | "transport"; +type MqTokenSigningMode = "none" | "hs256" | "rs256"; type JdbcDriverSelectItem = { id: string; label: string; @@ -132,6 +135,7 @@ const defaultForm = (): ConnectionForm => ({ redis_key_separator: ":", etcd_endpoints: "", gbase_server: "", + external_config: undefined, read_only: false, visible_databases: undefined, }); @@ -279,6 +283,24 @@ const dialogStep = ref("select"); const dbPickerView = ref("icon"); const dbSearchQuery = ref(""); const configTab = ref("connection"); +type MqAuthKind = MqAuth["kind"]; +const mqAdminUrl = ref("http://127.0.0.1:8080"); +const mqSystemKind = ref("pulsar"); +const mqAuthKind = ref("none"); +const mqToken = ref(""); +const mqBasicUsername = ref(""); +const mqBasicPassword = ref(""); +const mqApiKeyHeader = ref("Authorization"); +const mqApiKeyValue = ref(""); +const mqOauthIssuerUrl = ref(""); +const mqOauthClientId = ref(""); +const mqOauthClientSecret = ref(""); +const mqOauthAudience = ref(""); +const mqOauthScope = ref(""); +const mqTlsSkipVerify = ref(false); +const mqPinnedVersion = ref(pinnedVersionToSelection(undefined)); +const mqTokenSigningMode = ref("none"); +const mqTokenSigningKey = ref(""); const colorOptions = [ { value: "", class: "bg-transparent border-dashed", labelKey: "connection.colorNone" }, @@ -476,6 +498,7 @@ const driverProfiles: Record< xugu: { type: "xugu", port: 5138, user: "", label: "虚谷 XuguDB", icon: "xugu" }, iotdb: { type: "iotdb", port: 6667, user: "root", label: "Apache IoTDB", icon: "iotdb" }, etcd: { type: "etcd", port: 2379, user: "", label: "etcd", icon: "etcd" }, + mq: { type: "mq", port: 8080, user: "", label: "Apache Pulsar", icon: "pulsar", host: "127.0.0.1" }, iris: { type: "iris", port: 1972, user: "_SYSTEM", label: "IRIS", icon: "iris" }, influxdb: { type: "influxdb", port: 8086, user: "", label: "InfluxDB", icon: "InfluxDB" }, custom_mysql: { @@ -511,6 +534,104 @@ function selectedProfile() { return driverProfiles[selectedType.value] ?? driverProfiles.mysql; } +function resetMqFields(config?: Partial) { + mqSystemKind.value = "pulsar"; + mqAdminUrl.value = config?.adminUrl?.trim() || "http://127.0.0.1:8080"; + mqTlsSkipVerify.value = !!config?.tlsSkipVerify; + mqPinnedVersion.value = pinnedVersionToSelection(config?.pinnedVersion); + const auth = (config?.auth || { kind: "none" }) as MqAuth; + mqAuthKind.value = auth.kind || "none"; + mqToken.value = auth.token || ""; + mqBasicUsername.value = auth.username || ""; + mqBasicPassword.value = auth.password || ""; + mqApiKeyHeader.value = auth.header || "Authorization"; + mqApiKeyValue.value = auth.value || ""; + mqOauthIssuerUrl.value = auth.issuerUrl || ""; + mqOauthClientId.value = auth.clientId || ""; + mqOauthClientSecret.value = auth.clientSecret || ""; + mqOauthAudience.value = auth.audience || ""; + mqOauthScope.value = auth.scope || ""; + const tokenSigning = config?.tokenSigning; + mqTokenSigningMode.value = tokenSigning?.algorithm === "hs256" || tokenSigning?.algorithm === "rs256" ? tokenSigning.algorithm : "none"; + mqTokenSigningKey.value = tokenSigning?.key || ""; +} + +function hydrateMqFields(value: unknown) { + if (!value || typeof value !== "object") { + resetMqFields(); + return; + } + resetMqFields(value as Partial); +} + +function requireMqField(value: string, message: string): string { + const trimmed = value.trim(); + if (!trimmed) throw new Error(message); + return trimmed; +} + +function buildMqAuth(): MqAuth { + switch (mqAuthKind.value) { + case "token": + return { kind: "token", token: requireMqField(mqToken.value, "Token auth requires a token") }; + case "basic": + return { + kind: "basic", + username: requireMqField(mqBasicUsername.value, "Basic auth requires a username"), + password: mqBasicPassword.value, + }; + case "apiKey": + return { + kind: "apiKey", + header: requireMqField(mqApiKeyHeader.value, "API key auth requires a header"), + value: requireMqField(mqApiKeyValue.value, "API key auth requires a value"), + }; + case "oauth2": + return { + kind: "oauth2", + issuerUrl: requireMqField(mqOauthIssuerUrl.value, "OAuth2 auth requires an issuer URL"), + clientId: requireMqField(mqOauthClientId.value, "OAuth2 auth requires a client ID"), + clientSecret: requireMqField(mqOauthClientSecret.value, "OAuth2 auth requires a client secret"), + audience: mqOauthAudience.value.trim() || undefined, + scope: mqOauthScope.value.trim() || undefined, + }; + default: + return { kind: "none" }; + } +} + +function buildMqTokenSigning() { + if (mqTokenSigningMode.value === "none") return undefined; + return { + algorithm: mqTokenSigningMode.value, + key: requireMqField(mqTokenSigningKey.value, "Broker token signing key is required"), + }; +} + +function buildMqAdminConfig(): MqAdminConfig { + return { + systemKind: "pulsar", + adminUrl: requireMqField(mqAdminUrl.value, "MQ Admin URL is required"), + auth: buildMqAuth(), + tlsSkipVerify: mqTlsSkipVerify.value || undefined, + pinnedVersion: selectionToPinnedVersion(mqPinnedVersion.value), + tokenSigning: buildMqTokenSigning(), + }; +} + +function applyMqAdminUrl(config: LegacyConnectionConfig, adminUrl: string) { + let parsed: URL; + try { + parsed = new URL(adminUrl); + } catch { + throw new Error("MQ Admin URL is invalid"); + } + const port = Number(parsed.port) || (parsed.protocol === "https:" ? 443 : 8080); + config.host = parsed.hostname; + config.port = port; + config.ssl = parsed.protocol === "https:"; +} + function isCustomCompatibleProfile() { return selectedType.value === "custom_mysql" || selectedType.value === "custom_postgres"; } @@ -547,6 +668,11 @@ function applyProfile(val: string, preserveConnectionFields = false) { form.value.jdbc_driver_paths = []; jdbcDriverPathsInput.value = ""; } + if (profile.type === "mq") { + resetMqFields(); + form.value.database = undefined; + form.value.connection_string = undefined; + } } } @@ -613,6 +739,11 @@ watch( read_only: config.read_only || false, visible_databases: config.visible_databases, }; + if (config.db_type === "mq") { + hydrateMqFields(config.external_config); + } else { + resetMqFields(); + } h2ConnectionMode.value = h2ConnectionModeForConfig(config); customColorInput.value = config.color || ""; selectedTransportLayerId.value = form.value.transport_layers?.[0]?.id || null; @@ -634,6 +765,7 @@ watch( selectedTransportLayerId.value = null; selectedType.value = "mysql"; customDriverName.value = ""; + resetMqFields(); oceanbaseSubMode.value = "mysql"; h2ConnectionMode.value = "file"; dialogStep.value = "select"; @@ -767,6 +899,7 @@ const iconTypeMap: Record = { xugu: "xugu", iotdb: "iotdb", etcd: "etcd", + mq: "mq", dm: "dm", h2: "h2", snowflake: "snowflake", @@ -844,6 +977,7 @@ const dbOptions: DbOption[] = [ { value: "xugu", label: "虚谷 XuguDB" }, { value: "iotdb", label: "Apache IoTDB" }, { value: "etcd", label: "etcd" }, + { value: "mq", label: "Apache Pulsar" }, { value: "influxdb", label: "InfluxDB" }, { value: "iris", label: "IRIS" }, { value: "jdbc", label: "JDBC" }, @@ -996,6 +1130,7 @@ const testResultMessage = computed(() => { return testResult.value.ok ? t("connection.testSuccess") : testResult.value.message; }); const hasRequiredConnectionTarget = computed(() => { + if (form.value.db_type === "mq") return !!mqAdminUrl.value.trim(); if (isH2FileMode.value) return !!(form.value.host.trim() || h2FilePathFromJdbcUrl(form.value.connection_string)); return !!(form.value.host || (mongoUseUrl.value && form.value.connection_string) || (form.value.db_type === "jdbc" && form.value.connection_string) || connectionUrlInput.value.trim()); }); @@ -1121,6 +1256,18 @@ function connectionConfigForSubmit(id: string): ConnectionConfig { } if (!config.one_time) config.one_time = undefined; if (!config.read_only) config.read_only = undefined; + if (config.db_type === "mq") { + const mqConfig = buildMqAdminConfig(); + config.external_config = mqConfig; + applyMqAdminUrl(config, mqConfig.adminUrl); + config.username = ""; + config.password = ""; + config.database = undefined; + config.connection_string = undefined; + config.url_params = ""; + } else { + config.external_config = undefined; + } if (config.db_type === "mongodb" && !mongoUseUrl.value) { config.connection_string = undefined; } else if (config.db_type === "mongodb") { @@ -1575,6 +1722,7 @@ function resetForm() { selectedType.value = "mysql"; customDriverName.value = ""; mongoUseUrl.value = false; + resetMqFields(); oceanbaseSubMode.value = "mysql"; jdbcDriverPathsInput.value = ""; selectedJdbcDriverPath.value = ""; @@ -2439,6 +2587,124 @@ function openExternalUrl(url: string) { + + + + diff --git a/apps/desktop/src/components/mq/SubscriptionsPanel.vue b/apps/desktop/src/components/mq/SubscriptionsPanel.vue new file mode 100644 index 000000000..48cdeb584 --- /dev/null +++ b/apps/desktop/src/components/mq/SubscriptionsPanel.vue @@ -0,0 +1,848 @@ + + + + + diff --git a/apps/desktop/src/components/mq/TenantsPanel.vue b/apps/desktop/src/components/mq/TenantsPanel.vue new file mode 100644 index 000000000..3333c273f --- /dev/null +++ b/apps/desktop/src/components/mq/TenantsPanel.vue @@ -0,0 +1,760 @@ + + + + + diff --git a/apps/desktop/src/components/mq/TopicsPanel.vue b/apps/desktop/src/components/mq/TopicsPanel.vue new file mode 100644 index 000000000..ebf24cbf5 --- /dev/null +++ b/apps/desktop/src/components/mq/TopicsPanel.vue @@ -0,0 +1,612 @@ + + + + + diff --git a/apps/desktop/src/components/sidebar/ConnectionTree.vue b/apps/desktop/src/components/sidebar/ConnectionTree.vue index d5b33bed9..543d30790 100644 --- a/apps/desktop/src/components/sidebar/ConnectionTree.vue +++ b/apps/desktop/src/components/sidebar/ConnectionTree.vue @@ -79,7 +79,7 @@ const isFiltering = computed(() => !!searchQuery.value.trim() || hasSearchScopeF const SEARCH_SCOPE_TO_NODE_TYPES: Record = { connection: ["connection"], - database: ["database", "redis-db", "mongo-db"], + database: ["database", "redis-db", "mq-tenant", "mongo-db"], schema: ["schema"], table: ["table", "mongo-collection", "elasticsearch-index"], view: ["view"], @@ -334,6 +334,8 @@ async function ensureTreeLoadedForTarget(target: ActiveTabSidebarTarget, opts?: await store.loadMongoDatabases(connId); } else if (config.db_type === "elasticsearch") { await store.loadElasticsearchIndices(connId); + } else if (config.db_type === "mq") { + await store.loadMqTenants(connId, loadOptions); } else { await store.loadDatabases(connId, loadOptions); } @@ -342,6 +344,7 @@ async function ensureTreeLoadedForTarget(target: ActiveTabSidebarTarget, opts?: } } + if (config.db_type === "mq") return; if (!("database" in target) || !target.database) return; // Find the database node diff --git a/apps/desktop/src/components/sidebar/TreeItem.vue b/apps/desktop/src/components/sidebar/TreeItem.vue index 4e83abdb4..497bdb9cd 100644 --- a/apps/desktop/src/components/sidebar/TreeItem.vue +++ b/apps/desktop/src/components/sidebar/TreeItem.vue @@ -229,6 +229,8 @@ function getIconInfo(node: TreeNode): { icon: any; colorClass: string } | null { return { icon: Zap, colorClass: "text-orange-300" }; case "redis-db": return { icon: Database, colorClass: "text-red-400" }; + case "mq-tenant": + return { icon: FolderOpen, colorClass: "text-sky-400" }; case "etcd-root": return { icon: Database, colorClass: "text-sky-500" }; case "mongo-db": @@ -354,12 +356,16 @@ async function toggle() { await connectionStore.loadMongoDatabases(node.connectionId); } else if (config?.db_type === "elasticsearch") { await connectionStore.loadElasticsearchIndices(node.connectionId); + } else if (config?.db_type === "mq") { + await connectionStore.loadMqTenants(node.connectionId); } else { await connectionStore.loadDatabases(node.connectionId); } } else if (node.type === "redis-db" && node.connectionId && node.database) { const tabTitle = `${connectionStore.getConfig(node.connectionId)?.name || "Redis"}:db${node.database}`; queryStore.createTab(node.connectionId, node.database, tabTitle, "redis"); + } else if (node.type === "mq-tenant" && node.connectionId) { + queryStore.openMqAdmin(node.connectionId, { tenant: node.mqTenant || node.label }); } else if (node.type === "etcd-root" && node.connectionId) { const tabTitle = `${connectionStore.getConfig(node.connectionId)?.name || "etcd"}:keys`; queryStore.createTab(node.connectionId, "", tabTitle, "etcd"); @@ -2620,7 +2626,7 @@ const nodeIconClass = computed(() => { const canConfigureVisibleDatabases = computed(() => { if (props.node.type !== "connection" || !props.node.connectionId) return false; const dbType = connectionStore.getConfig(props.node.connectionId)?.db_type; - return dbType !== "elasticsearch" && dbType !== "etcd"; + return dbType !== "elasticsearch" && dbType !== "etcd" && dbType !== "mq"; }); const canCopyFinalProxyPort = computed(() => { if (props.node.type !== "connection" || !props.node.connectionId) return false; diff --git a/apps/desktop/src/lib/__tests__/connectionOpenTarget.spec.ts b/apps/desktop/src/lib/__tests__/connectionOpenTarget.spec.ts new file mode 100644 index 000000000..b255fd70a --- /dev/null +++ b/apps/desktop/src/lib/__tests__/connectionOpenTarget.spec.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { quickConnectionOpenTarget } from "@/lib/connectionOpenTarget"; +import type { ConnectionConfig } from "@/types/database"; + +function connection(dbType: ConnectionConfig["db_type"]): ConnectionConfig { + return { + id: "conn", + name: "conn", + db_type: dbType, + host: "127.0.0.1", + port: 0, + user: "", + password: "", + database: "", + readonly: false, + read_only: false, + ssl_mode: "disabled", + color: "#888", + } as ConnectionConfig; +} + +describe("quickConnectionOpenTarget", () => { + it("opens message queue connections in the MQ admin console", () => { + expect(quickConnectionOpenTarget(connection("mq"))).toEqual({ kind: "mq-admin" }); + }); + + it("opens regular connections in a query tab", () => { + expect(quickConnectionOpenTarget({ ...connection("postgresql"), database: "app" })).toEqual({ + kind: "query", + database: "app", + }); + }); +}); diff --git a/apps/desktop/src/lib/__tests__/mqListPanels.spec.ts b/apps/desktop/src/lib/__tests__/mqListPanels.spec.ts new file mode 100644 index 000000000..5f80520f4 --- /dev/null +++ b/apps/desktop/src/lib/__tests__/mqListPanels.spec.ts @@ -0,0 +1,45 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +function componentSource(name: string): string { + return readFileSync(new URL(`../../components/mq/${name}`, import.meta.url), "utf8"); +} + +describe("MQ list panels", () => { + it("does not issue per-tenant detail requests from the tenant list UI", () => { + const source = componentSource("TenantsPanel.vue"); + + expect(source).not.toContain("mqGetTenant"); + }); + + it("does not issue per-namespace permission requests from the namespace list UI", () => { + const source = componentSource("NamespacesPanel.vue"); + + expect(source).not.toContain("mqListPermissions"); + }); + + it("refreshing policies hydrates missing fields from defaults instead of dirty form values", () => { + const source = componentSource("PoliciesPanel.vue"); + + expect(source).not.toContain("policyFormsFromEffectivePolicies(loaded, currentPolicyForms())"); + expect(source).toContain("policyFormsFromEffectivePolicies(loaded, defaultMqPolicyForms())"); + }); + + it("raw api panel includes common endpoint presets that use the selected mq context", () => { + const rawSource = componentSource("RawApiPanel.vue"); + const consoleSource = componentSource("MqAdminConsole.vue"); + + expect(rawSource).toContain("const presets = computed"); + expect(rawSource).toContain("/admin/v2/brokers/version"); + expect(rawSource).toContain("/internalStats"); + expect(rawSource).toContain("/partitioned-stats"); + expect(rawSource).toContain("/schema"); + expect(rawSource).toContain("presetsCollapsed"); + expect(rawSource).toContain("formatJsonBody"); + expect(rawSource).toContain("bodyTextareaRows"); + expect(rawSource).toContain("body: isReadMethod.value ? undefined : parseBody()"); + expect(consoleSource).toContain(':tenant="selectedTenant"'); + expect(consoleSource).toContain(':namespace="selectedNamespace"'); + expect(consoleSource).toContain(':topic="selectedTopic"'); + }); +}); diff --git a/apps/desktop/src/lib/__tests__/mqPinnedVersionOptions.spec.ts b/apps/desktop/src/lib/__tests__/mqPinnedVersionOptions.spec.ts new file mode 100644 index 000000000..7d9bf835e --- /dev/null +++ b/apps/desktop/src/lib/__tests__/mqPinnedVersionOptions.spec.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { MQ_PINNED_VERSION_AUTO, MQ_PINNED_VERSION_OPTIONS, pinnedVersionToSelection, selectionToPinnedVersion } from "@/lib/mqPinnedVersionOptions"; + +describe("mqPinnedVersionOptions", () => { + it("offers auto-detection and the supported Pulsar 3.x profile", () => { + expect(MQ_PINNED_VERSION_OPTIONS.map((option) => option.value)).toEqual([MQ_PINNED_VERSION_AUTO, "3.1.x"]); + }); + + it("normalizes existing Pulsar 3.x pinned values to the supported profile option", () => { + expect(pinnedVersionToSelection(undefined)).toBe(MQ_PINNED_VERSION_AUTO); + expect(pinnedVersionToSelection("")).toBe(MQ_PINNED_VERSION_AUTO); + expect(pinnedVersionToSelection("3.0.7")).toBe("3.1.x"); + expect(pinnedVersionToSelection("3.1.2")).toBe("3.1.x"); + expect(pinnedVersionToSelection("3.1.x")).toBe("3.1.x"); + }); + + it("only persists an explicit version when a supported profile is selected", () => { + expect(selectionToPinnedVersion(MQ_PINNED_VERSION_AUTO)).toBeUndefined(); + expect(selectionToPinnedVersion("3.1.x")).toBe("3.1.x"); + expect(selectionToPinnedVersion("4.0.x")).toBeUndefined(); + }); +}); diff --git a/apps/desktop/src/lib/__tests__/mqPolicyForms.spec.ts b/apps/desktop/src/lib/__tests__/mqPolicyForms.spec.ts new file mode 100644 index 000000000..c38c3b5e4 --- /dev/null +++ b/apps/desktop/src/lib/__tests__/mqPolicyForms.spec.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { defaultMqPolicyForms, policyFormsFromEffectivePolicies } from "@/lib/mqPolicyForms"; + +describe("mqPolicyForms", () => { + it("hydrates forms from topic effective policies", () => { + const forms = policyFormsFromEffectivePolicies( + { + level: "topic", + topicPolicies: { + publishRate: { + publishThrottlingRateInMsg: 100, + publishThrottlingRateInByte: 2048, + }, + dispatchRate: { + dispatchThrottlingRateInMsg: 50, + dispatchThrottlingRateInByte: 1024, + ratePeriodInSecond: 5, + }, + backlogQuota: { + limitSize: 4096, + limitTime: 60, + policy: "producer_exception", + quotaType: "message_age", + }, + retention: { + retentionTimeInMinutes: 120, + retentionSizeInMB: 512, + }, + }, + }, + defaultMqPolicyForms(), + ); + + expect(forms.publishForm.publishThrottlingRateInMsg).toBe(100); + expect(forms.dispatchForm.ratePeriodInSecond).toBe(5); + expect(forms.backlogForm.quotaType).toBe("message_age"); + expect(forms.retentionForm.retentionSizeInMb).toBe(512); + }); + + it("hydrates forms from namespace Pulsar policies shape", () => { + const forms = policyFormsFromEffectivePolicies( + { + publishRate: { + publishThrottlingRateInMsg: 20, + publishThrottlingRateInByte: 3000, + }, + subscribeRate: { + subscribeThrottlingRatePerConsumer: 12, + ratePeriodInSecond: 3, + }, + backlog_quota_map: { + destination_storage: { + limitSize: 999, + limitTime: 10, + policy: "consumer_backlog_eviction", + }, + }, + retention_policies: { + retentionTimeInMinutes: 30, + retentionSizeInMB: 64, + }, + }, + defaultMqPolicyForms(), + ); + + expect(forms.publishForm.publishThrottlingRateInByte).toBe(3000); + expect(forms.subscribeForm.ratePeriodInSecond).toBe(3); + expect(forms.backlogForm.limitSize).toBe(999); + expect(forms.backlogForm.quotaType).toBe("destination_storage"); + expect(forms.retentionForm.retentionTimeInMinutes).toBe(30); + }); +}); diff --git a/apps/desktop/src/lib/__tests__/mqTenantForm.spec.ts b/apps/desktop/src/lib/__tests__/mqTenantForm.spec.ts new file mode 100644 index 000000000..cb120d3d9 --- /dev/null +++ b/apps/desktop/src/lib/__tests__/mqTenantForm.spec.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { defaultTenantConfig, mqClusterOptionsFromExtra, validateTenantForm } from "@/lib/mqTenantForm"; + +describe("mqTenantForm", () => { + it("extracts selectable cluster names from Pulsar cluster info extras", () => { + expect(mqClusterOptionsFromExtra({ clusters: ["standalone", "ec-pulsar", "standalone", "", 42] })).toEqual(["standalone", "ec-pulsar"]); + expect(mqClusterOptionsFromExtra({ clusters: "standalone" })).toEqual([]); + expect(mqClusterOptionsFromExtra(undefined)).toEqual([]); + }); + + it("prefills a new tenant with detected clusters", () => { + const config = defaultTenantConfig(["standalone", "ec-pulsar"]); + + expect(config).toEqual({ + adminRoles: [], + allowedClusters: ["standalone", "ec-pulsar"], + }); + }); + + it("requires both a tenant name and at least one allowed cluster", () => { + expect(validateTenantForm("", { adminRoles: [], allowedClusters: ["standalone"] })).toBe("Tenant name is required"); + expect(validateTenantForm("tenant-a", { adminRoles: [], allowedClusters: [] })).toBe("Allowed clusters are required"); + expect(validateTenantForm("tenant-a", { adminRoles: [], allowedClusters: ["standalone"] })).toBeUndefined(); + }); +}); diff --git a/apps/desktop/src/lib/__tests__/mqTokenErrors.spec.ts b/apps/desktop/src/lib/__tests__/mqTokenErrors.spec.ts new file mode 100644 index 000000000..546eaafb2 --- /dev/null +++ b/apps/desktop/src/lib/__tests__/mqTokenErrors.spec.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { formatMqTokenIssueError } from "../mqTokenErrors"; + +describe("MQ token errors", () => { + it("turns missing signing configuration backend errors into actionable Chinese copy", () => { + const result = formatMqTokenIssueError("/api/mq/tokens/issue returned 500: Token signing is not configured for this MQ connection"); + + expect(result.kind).toBe("missingSigningKey"); + expect(result.title).toBe("未配置 Token 签发密钥"); + expect(result.message).toContain("无法生成客户端 Token"); + expect(result.detail).toContain("Broker Token 签发"); + }); + + it("keeps unrelated errors as ordinary messages", () => { + const result = formatMqTokenIssueError("network failed"); + + expect(result.kind).toBe("generic"); + expect(result.title).toBeUndefined(); + expect(result.message).toBe("network failed"); + }); +}); diff --git a/apps/desktop/src/lib/api.ts b/apps/desktop/src/lib/api.ts index bfcb15d64..52f077cf2 100644 --- a/apps/desktop/src/lib/api.ts +++ b/apps/desktop/src/lib/api.ts @@ -275,6 +275,48 @@ export const etcdGet = forward("etcdGet"); export const etcdPut = forward("etcdPut"); export const etcdDelete = forward("etcdDelete"); +// Message Queue +export const mqTestConnection = forward("mqTestConnection"); +export const mqListTenants = forward("mqListTenants"); +export const mqGetTenant = forward("mqGetTenant"); +export const mqCreateTenant = forward("mqCreateTenant"); +export const mqUpdateTenant = forward("mqUpdateTenant"); +export const mqDeleteTenant = forward("mqDeleteTenant"); +export const mqListNamespaces = forward("mqListNamespaces"); +export const mqCreateNamespace = forward("mqCreateNamespace"); +export const mqDeleteNamespace = forward("mqDeleteNamespace"); +export const mqGetNamespacePolicies = forward("mqGetNamespacePolicies"); +export const mqListTopics = forward("mqListTopics"); +export const mqCreateTopic = forward("mqCreateTopic"); +export const mqDeleteTopic = forward("mqDeleteTopic"); +export const mqUpdatePartitions = forward("mqUpdatePartitions"); +export const mqGetTopicStats = forward("mqGetTopicStats"); +export const mqGetTopicInternalStats = forward("mqGetTopicInternalStats"); +export const mqListSubscriptions = forward("mqListSubscriptions"); +export const mqCreateSubscription = forward("mqCreateSubscription"); +export const mqDeleteSubscription = forward("mqDeleteSubscription"); +export const mqSkipMessages = forward("mqSkipMessages"); +export const mqResetCursor = forward("mqResetCursor"); +export const mqClearBacklog = forward("mqClearBacklog"); +export const mqPeekMessages = forward("mqPeekMessages"); +export const mqExpireMessages = forward("mqExpireMessages"); +export const mqListProducers = forward("mqListProducers"); +export const mqListConsumers = forward("mqListConsumers"); +export const mqUnloadTopic = forward("mqUnloadTopic"); +export const mqSetPublishRate = forward("mqSetPublishRate"); +export const mqSetDispatchRate = forward("mqSetDispatchRate"); +export const mqSetSubscribeRate = forward("mqSetSubscribeRate"); +export const mqSetBacklogQuota = forward("mqSetBacklogQuota"); +export const mqSetRetention = forward("mqSetRetention"); +export const mqGetEffectivePolicies = forward("mqGetEffectivePolicies"); +export const mqGrantPermission = forward("mqGrantPermission"); +export const mqRevokePermission = forward("mqRevokePermission"); +export const mqListPermissions = forward("mqListPermissions"); +export const mqIssueToken = forward("mqIssueToken"); +export const mqListTokenRecords = forward("mqListTokenRecords"); +export const mqGetBacklog = forward("mqGetBacklog"); +export const mqRawRequest = forward("mqRawRequest"); + // MongoDB export const mongoListDatabases = forward("mongoListDatabases"); export const mongoListCollections = forward("mongoListCollections"); diff --git a/apps/desktop/src/lib/connectionOpenTarget.ts b/apps/desktop/src/lib/connectionOpenTarget.ts new file mode 100644 index 000000000..af84df3ed --- /dev/null +++ b/apps/desktop/src/lib/connectionOpenTarget.ts @@ -0,0 +1,11 @@ +import type { ConnectionConfig } from "@/types/database"; +import { resolveDefaultDatabase } from "@/lib/defaultDatabase"; + +export type QuickConnectionOpenTarget = { kind: "mq-admin" } | { kind: "query"; database: string }; + +export function quickConnectionOpenTarget(connection: Pick, databaseOptions: string[] = []): QuickConnectionOpenTarget { + if (connection.db_type === "mq") { + return { kind: "mq-admin" }; + } + return { kind: "query", database: resolveDefaultDatabase(connection, databaseOptions) }; +} diff --git a/apps/desktop/src/lib/errorUtils.ts b/apps/desktop/src/lib/errorUtils.ts new file mode 100644 index 000000000..bec50ff52 --- /dev/null +++ b/apps/desktop/src/lib/errorUtils.ts @@ -0,0 +1,63 @@ +/** + * Utility functions for consistent error handling across the application. + */ + +/** + * Formats an unknown error value into a human-readable string. + * Handles Error objects, strings, null/undefined, and other types. + * + * @param e - The error value to format (from a catch block) + * @returns A human-readable error message string + * + * @example + * try { + * await someOperation(); + * } catch (e: unknown) { + * errorMessage.value = formatError(e); + * } + */ +export function formatError(e: unknown): string { + if (e instanceof Error) { + return e.message; + } + + if (typeof e === "string") { + return e; + } + + if (e === null || e === undefined) { + return "Unknown error occurred"; + } + + // Try to extract message property from object-like values + if (typeof e === "object" && "message" in e) { + const message = (e as { message: unknown }).message; + if (typeof message === "string") { + return message; + } + } + + // Fallback: attempt to stringify + try { + return String(e); + } catch { + return "Unknown error occurred"; + } +} + +/** + * Formats an error with a context prefix for better debugging. + * + * @param e - The error value to format + * @param context - The operation context (e.g., "loading topics", "creating tenant") + * @returns A formatted error message with context + * + * @example + * catch (e: unknown) { + * errorMessage.value = formatErrorWithContext(e, 'loading topics'); + * } + */ +export function formatErrorWithContext(e: unknown, context: string): string { + const message = formatError(e); + return `Failed to ${context}: ${message}`; +} diff --git a/apps/desktop/src/lib/http.ts b/apps/desktop/src/lib/http.ts index d4367fe7c..a6447450e 100644 --- a/apps/desktop/src/lib/http.ts +++ b/apps/desktop/src/lib/http.ts @@ -1512,5 +1512,7 @@ export async function loadSidebarLayout(): Promise { } export async function refreshConnections(): Promise { - // Web mode doesn't maintain persistent connection pools 鈥?no-op + // Web mode doesn't maintain persistent connection pools - no-op } + +export * from "./mq-http"; diff --git a/apps/desktop/src/lib/mq-http.ts b/apps/desktop/src/lib/mq-http.ts new file mode 100644 index 000000000..4a503658f --- /dev/null +++ b/apps/desktop/src/lib/mq-http.ts @@ -0,0 +1,206 @@ +// HTTP fetch API for message queue admin (web mode) +import type { + MqClusterInfo, + TenantInfo, + TenantConfig, + NamespaceRef, + NamespaceInfo, + NamespaceConfig, + TopicRef, + TopicInfo, + ListTopicsOpts, + TopicStats, + SubscriptionInfo, + ResetPosition, + SkipCount, + ConsumerInfo, + ProducerInfo, + PolicyScope, + PublishRate, + DispatchRate, + SubscribeRate, + BacklogQuota, + RetentionPolicy, + AuthAction, + PermissionMap, + MqTokenIssueRequest, + MqTokenRecord, + MqIssuedToken, + BacklogStats, + PeekedMessage, + MqRawRequest, + MqRawResponse, +} from "@/types/mq"; + +async function post(path: string, body: unknown): Promise { + const resp = await fetch(path, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!resp.ok) { + const detail = (await resp.text().catch(() => "")).trim(); + throw new Error(detail ? `${path} returned ${resp.status}: ${detail}` : `${path} returned ${resp.status}`); + } + return resp.json(); +} + +export async function mqTestConnection(connectionId: string): Promise { + return post("/api/mq/test-connection", { connectionId }); +} + +export async function mqListTenants(connectionId: string): Promise { + return post("/api/mq/tenants/list", { connectionId }); +} + +export async function mqGetTenant(connectionId: string, name: string): Promise { + return post("/api/mq/tenants/get", { connectionId, name }); +} + +export async function mqCreateTenant(connectionId: string, name: string, config: TenantConfig): Promise { + return post("/api/mq/tenants/create", { connectionId, name, config }); +} + +export async function mqUpdateTenant(connectionId: string, name: string, config: TenantConfig): Promise { + return post("/api/mq/tenants/update", { connectionId, name, config }); +} + +export async function mqDeleteTenant(connectionId: string, name: string, force: boolean): Promise { + return post("/api/mq/tenants/delete", { connectionId, name, force }); +} + +export async function mqListNamespaces(connectionId: string, tenant: string): Promise { + return post("/api/mq/namespaces/list", { connectionId, tenant }); +} + +export async function mqCreateNamespace(connectionId: string, ns: NamespaceRef, config: NamespaceConfig): Promise { + return post("/api/mq/namespaces/create", { connectionId, ns, config }); +} + +export async function mqDeleteNamespace(connectionId: string, ns: NamespaceRef, force: boolean): Promise { + return post("/api/mq/namespaces/delete", { connectionId, ns, force }); +} + +export async function mqGetNamespacePolicies(connectionId: string, ns: NamespaceRef): Promise { + return post("/api/mq/namespaces/policies", { connectionId, ns }); +} + +export async function mqListTopics(connectionId: string, ns: NamespaceRef, opts: ListTopicsOpts): Promise { + return post("/api/mq/topics/list", { connectionId, ns, opts }); +} + +export async function mqCreateTopic(connectionId: string, topic: TopicRef, partitions?: number): Promise { + return post("/api/mq/topics/create", { connectionId, topic, partitions }); +} + +export async function mqDeleteTopic(connectionId: string, topic: TopicRef, force: boolean): Promise { + return post("/api/mq/topics/delete", { connectionId, topic, force }); +} + +export async function mqUpdatePartitions(connectionId: string, topic: TopicRef, partitions: number): Promise { + return post("/api/mq/topics/update-partitions", { connectionId, topic, partitions }); +} + +export async function mqGetTopicStats(connectionId: string, topic: TopicRef): Promise { + return post("/api/mq/topics/stats", { connectionId, topic }); +} + +export async function mqGetTopicInternalStats(connectionId: string, topic: TopicRef): Promise { + return post("/api/mq/topics/internal-stats", { connectionId, topic }); +} + +export async function mqListSubscriptions(connectionId: string, topic: TopicRef): Promise { + return post("/api/mq/subscriptions/list", { connectionId, topic }); +} + +export async function mqCreateSubscription(connectionId: string, topic: TopicRef, sub: string, pos: ResetPosition): Promise { + return post("/api/mq/subscriptions/create", { connectionId, topic, sub, pos }); +} + +export async function mqDeleteSubscription(connectionId: string, topic: TopicRef, sub: string, force: boolean): Promise { + return post("/api/mq/subscriptions/delete", { connectionId, topic, sub, force }); +} + +export async function mqSkipMessages(connectionId: string, topic: TopicRef, sub: string, count: SkipCount): Promise { + return post("/api/mq/subscriptions/skip-messages", { connectionId, topic, sub, count }); +} + +export async function mqResetCursor(connectionId: string, topic: TopicRef, sub: string, pos: ResetPosition): Promise { + return post("/api/mq/subscriptions/reset-cursor", { connectionId, topic, sub, pos }); +} + +export async function mqClearBacklog(connectionId: string, topic: TopicRef, sub: string): Promise { + return post("/api/mq/subscriptions/clear-backlog", { connectionId, topic, sub }); +} + +export async function mqPeekMessages(connectionId: string, topic: TopicRef, sub: string, count: number): Promise { + return post("/api/mq/subscriptions/peek-messages", { connectionId, topic, sub, count }); +} + +export async function mqExpireMessages(connectionId: string, topic: TopicRef, sub: string, expireSeconds: number): Promise { + return post("/api/mq/subscriptions/expire-messages", { connectionId, topic, sub, expireSeconds }); +} + +export async function mqListProducers(connectionId: string, topic: TopicRef): Promise { + return post("/api/mq/producers/list", { connectionId, topic }); +} + +export async function mqListConsumers(connectionId: string, topic: TopicRef, sub: string): Promise { + return post("/api/mq/consumers/list", { connectionId, topic, sub }); +} + +export async function mqUnloadTopic(connectionId: string, topic: TopicRef): Promise { + return post("/api/mq/topics/unload", { connectionId, topic }); +} + +export async function mqSetPublishRate(connectionId: string, scope: PolicyScope, rate: PublishRate): Promise { + return post("/api/mq/policies/publish-rate", { connectionId, scope, rate }); +} + +export async function mqSetDispatchRate(connectionId: string, scope: PolicyScope, rate: DispatchRate): Promise { + return post("/api/mq/policies/dispatch-rate", { connectionId, scope, rate }); +} + +export async function mqSetSubscribeRate(connectionId: string, scope: PolicyScope, rate: SubscribeRate): Promise { + return post("/api/mq/policies/subscribe-rate", { connectionId, scope, rate }); +} + +export async function mqSetBacklogQuota(connectionId: string, scope: PolicyScope, quota: BacklogQuota): Promise { + return post("/api/mq/policies/backlog-quota", { connectionId, scope, quota }); +} + +export async function mqSetRetention(connectionId: string, scope: PolicyScope, retention: RetentionPolicy): Promise { + return post("/api/mq/policies/retention", { connectionId, scope, retention }); +} + +export async function mqGetEffectivePolicies(connectionId: string, scope: PolicyScope): Promise { + return post("/api/mq/policies/effective", { connectionId, scope }); +} + +export async function mqGrantPermission(connectionId: string, scope: PolicyScope, role: string, actions: AuthAction[]): Promise { + return post("/api/mq/permissions/grant", { connectionId, scope, role, actions }); +} + +export async function mqRevokePermission(connectionId: string, scope: PolicyScope, role: string): Promise { + return post("/api/mq/permissions/revoke", { connectionId, scope, role }); +} + +export async function mqListPermissions(connectionId: string, scope: PolicyScope): Promise { + return post("/api/mq/permissions/list", { connectionId, scope }); +} + +export async function mqIssueToken(connectionId: string, req: MqTokenIssueRequest): Promise { + return post("/api/mq/tokens/issue", { connectionId, req }); +} + +export async function mqListTokenRecords(connectionId: string, subject?: string): Promise { + return post("/api/mq/tokens/list", { connectionId, subject }); +} + +export async function mqGetBacklog(connectionId: string, topic: TopicRef, sub?: string): Promise { + return post("/api/mq/monitoring/backlog", { connectionId, topic, sub }); +} + +export async function mqRawRequest(connectionId: string, req: MqRawRequest): Promise { + return post("/api/mq/raw", { connectionId, req }); +} diff --git a/apps/desktop/src/lib/mq-tauri.ts b/apps/desktop/src/lib/mq-tauri.ts new file mode 100644 index 000000000..f726e4f99 --- /dev/null +++ b/apps/desktop/src/lib/mq-tauri.ts @@ -0,0 +1,204 @@ +// Message queue admin API functions - Tauri invoke layer +import { invoke } from "@tauri-apps/api/core"; +import type { + MqClusterInfo, + TenantInfo, + TenantConfig, + NamespaceRef, + NamespaceInfo, + NamespaceConfig, + TopicRef, + TopicInfo, + ListTopicsOpts, + TopicStats, + SubscriptionInfo, + ResetPosition, + SkipCount, + ConsumerInfo, + ProducerInfo, + PolicyScope, + PublishRate, + DispatchRate, + SubscribeRate, + BacklogQuota, + RetentionPolicy, + AuthAction, + PermissionMap, + MqTokenIssueRequest, + MqTokenRecord, + MqIssuedToken, + BacklogStats, + PeekedMessage, + MqRawRequest, + MqRawResponse, +} from "@/types/mq"; + +// Connectivity +export async function mqTestConnection(connectionId: string): Promise { + return invoke("mq_test_connection", { connectionId }); +} + +// Tenants +export async function mqListTenants(connectionId: string): Promise { + return invoke("mq_list_tenants", { connectionId }); +} + +export async function mqGetTenant(connectionId: string, name: string): Promise { + return invoke("mq_get_tenant", { connectionId, name }); +} + +export async function mqCreateTenant(connectionId: string, name: string, config: TenantConfig): Promise { + return invoke("mq_create_tenant", { connectionId, name, config }); +} + +export async function mqUpdateTenant(connectionId: string, name: string, config: TenantConfig): Promise { + return invoke("mq_update_tenant", { connectionId, name, config }); +} + +export async function mqDeleteTenant(connectionId: string, name: string, force: boolean): Promise { + return invoke("mq_delete_tenant", { connectionId, name, force }); +} + +// Namespaces +export async function mqListNamespaces(connectionId: string, tenant: string): Promise { + return invoke("mq_list_namespaces", { connectionId, tenant }); +} + +export async function mqCreateNamespace(connectionId: string, ns: NamespaceRef, config: NamespaceConfig): Promise { + return invoke("mq_create_namespace", { connectionId, ns, config }); +} + +export async function mqDeleteNamespace(connectionId: string, ns: NamespaceRef, force: boolean): Promise { + return invoke("mq_delete_namespace", { connectionId, ns, force }); +} + +export async function mqGetNamespacePolicies(connectionId: string, ns: NamespaceRef): Promise { + return invoke("mq_get_namespace_policies", { connectionId, ns }); +} + +// Topics +export async function mqListTopics(connectionId: string, ns: NamespaceRef, opts: ListTopicsOpts): Promise { + return invoke("mq_list_topics", { connectionId, ns, opts }); +} + +export async function mqCreateTopic(connectionId: string, topic: TopicRef, partitions?: number): Promise { + return invoke("mq_create_topic", { connectionId, topic, partitions }); +} + +export async function mqDeleteTopic(connectionId: string, topic: TopicRef, force: boolean): Promise { + return invoke("mq_delete_topic", { connectionId, topic, force }); +} + +export async function mqUpdatePartitions(connectionId: string, topic: TopicRef, partitions: number): Promise { + return invoke("mq_update_partitions", { connectionId, topic, partitions }); +} + +export async function mqGetTopicStats(connectionId: string, topic: TopicRef): Promise { + return invoke("mq_get_topic_stats", { connectionId, topic }); +} + +export async function mqGetTopicInternalStats(connectionId: string, topic: TopicRef): Promise { + return invoke("mq_get_topic_internal_stats", { connectionId, topic }); +} + +// Subscriptions +export async function mqListSubscriptions(connectionId: string, topic: TopicRef): Promise { + return invoke("mq_list_subscriptions", { connectionId, topic }); +} + +export async function mqCreateSubscription(connectionId: string, topic: TopicRef, sub: string, pos: ResetPosition): Promise { + return invoke("mq_create_subscription", { connectionId, topic, sub, pos }); +} + +export async function mqDeleteSubscription(connectionId: string, topic: TopicRef, sub: string, force: boolean): Promise { + return invoke("mq_delete_subscription", { connectionId, topic, sub, force }); +} + +export async function mqSkipMessages(connectionId: string, topic: TopicRef, sub: string, count: SkipCount): Promise { + return invoke("mq_skip_messages", { connectionId, topic, sub, count }); +} + +export async function mqResetCursor(connectionId: string, topic: TopicRef, sub: string, pos: ResetPosition): Promise { + return invoke("mq_reset_cursor", { connectionId, topic, sub, pos }); +} + +export async function mqClearBacklog(connectionId: string, topic: TopicRef, sub: string): Promise { + return invoke("mq_clear_backlog", { connectionId, topic, sub }); +} + +export async function mqPeekMessages(connectionId: string, topic: TopicRef, sub: string, count: number): Promise { + return invoke("mq_peek_messages", { connectionId, topic, sub, count }); +} + +export async function mqExpireMessages(connectionId: string, topic: TopicRef, sub: string, expireSeconds: number): Promise { + return invoke("mq_expire_messages", { connectionId, topic, sub, expireSeconds }); +} + +// Producers / Consumers +export async function mqListProducers(connectionId: string, topic: TopicRef): Promise { + return invoke("mq_list_producers", { connectionId, topic }); +} + +export async function mqListConsumers(connectionId: string, topic: TopicRef, sub: string): Promise { + return invoke("mq_list_consumers", { connectionId, topic, sub }); +} + +export async function mqUnloadTopic(connectionId: string, topic: TopicRef): Promise { + return invoke("mq_unload_topic", { connectionId, topic }); +} + +// Policies +export async function mqSetPublishRate(connectionId: string, scope: PolicyScope, rate: PublishRate): Promise { + return invoke("mq_set_publish_rate", { connectionId, scope, rate }); +} + +export async function mqSetDispatchRate(connectionId: string, scope: PolicyScope, rate: DispatchRate): Promise { + return invoke("mq_set_dispatch_rate", { connectionId, scope, rate }); +} + +export async function mqSetSubscribeRate(connectionId: string, scope: PolicyScope, rate: SubscribeRate): Promise { + return invoke("mq_set_subscribe_rate", { connectionId, scope, rate }); +} + +export async function mqSetBacklogQuota(connectionId: string, scope: PolicyScope, quota: BacklogQuota): Promise { + return invoke("mq_set_backlog_quota", { connectionId, scope, quota }); +} + +export async function mqSetRetention(connectionId: string, scope: PolicyScope, retention: RetentionPolicy): Promise { + return invoke("mq_set_retention", { connectionId, scope, retention }); +} + +export async function mqGetEffectivePolicies(connectionId: string, scope: PolicyScope): Promise { + return invoke("mq_get_effective_policies", { connectionId, scope }); +} + +// Permissions +export async function mqGrantPermission(connectionId: string, scope: PolicyScope, role: string, actions: AuthAction[]): Promise { + return invoke("mq_grant_permission", { connectionId, scope, role, actions }); +} + +export async function mqRevokePermission(connectionId: string, scope: PolicyScope, role: string): Promise { + return invoke("mq_revoke_permission", { connectionId, scope, role }); +} + +export async function mqListPermissions(connectionId: string, scope: PolicyScope): Promise { + return invoke("mq_list_permissions", { connectionId, scope }); +} + +export async function mqIssueToken(connectionId: string, req: MqTokenIssueRequest): Promise { + return invoke("mq_issue_token", { connectionId, req }); +} + +export async function mqListTokenRecords(connectionId: string, subject?: string): Promise { + return invoke("mq_list_token_records", { connectionId, subject }); +} + +// Monitoring +export async function mqGetBacklog(connectionId: string, topic: TopicRef, sub?: string): Promise { + return invoke("mq_get_backlog", { connectionId, topic, sub }); +} + +// Raw request +export async function mqRawRequest(connectionId: string, req: MqRawRequest): Promise { + return invoke("mq_raw_request", { connectionId, req }); +} diff --git a/apps/desktop/src/lib/mqPinnedVersionOptions.ts b/apps/desktop/src/lib/mqPinnedVersionOptions.ts new file mode 100644 index 000000000..d8bbcf9cf --- /dev/null +++ b/apps/desktop/src/lib/mqPinnedVersionOptions.ts @@ -0,0 +1,31 @@ +export const MQ_PINNED_VERSION_AUTO = "auto"; + +export interface MqPinnedVersionOption { + value: string; + label: string; + description: string; +} + +export const MQ_PINNED_VERSION_OPTIONS: MqPinnedVersionOption[] = [ + { + value: MQ_PINNED_VERSION_AUTO, + label: "Auto detect", + description: "Probe /admin/v2/brokers/version and use the default 3.1.x profile if probing fails.", + }, + { + value: "3.1.x", + label: "Apache Pulsar 3.1.x", + description: "Supported Pulsar 3.x admin API profile, compatible with the 3.0/3.1 LTS line.", + }, +]; + +export function pinnedVersionToSelection(pinnedVersion: string | null | undefined): string { + const value = pinnedVersion?.trim(); + if (!value) return MQ_PINNED_VERSION_AUTO; + if (/^3(?:\.|$)/.test(value)) return "3.1.x"; + return MQ_PINNED_VERSION_AUTO; +} + +export function selectionToPinnedVersion(selection: string): string | undefined { + return MQ_PINNED_VERSION_OPTIONS.some((option) => option.value === selection) && selection !== MQ_PINNED_VERSION_AUTO ? selection : undefined; +} diff --git a/apps/desktop/src/lib/mqPolicyForms.ts b/apps/desktop/src/lib/mqPolicyForms.ts new file mode 100644 index 000000000..8260f3e45 --- /dev/null +++ b/apps/desktop/src/lib/mqPolicyForms.ts @@ -0,0 +1,137 @@ +import type { BacklogQuota, DispatchRate, PublishRate, RetentionPolicy, SubscribeRate } from "@/types/mq"; + +export interface MqPolicyForms { + publishForm: PublishRate; + dispatchForm: DispatchRate; + subscribeForm: SubscribeRate; + backlogForm: BacklogQuota; + retentionForm: RetentionPolicy; +} + +export function defaultMqPolicyForms(): MqPolicyForms { + return { + publishForm: { + publishThrottlingRateInMsg: 0, + publishThrottlingRateInByte: 0, + }, + dispatchForm: { + dispatchThrottlingRateInMsg: 0, + dispatchThrottlingRateInByte: 0, + ratePeriodInSecond: 1, + }, + subscribeForm: { + subscribeThrottlingRatePerConsumer: 0, + ratePeriodInSecond: 1, + }, + backlogForm: { + limitSize: -1, + limitTime: -1, + policy: "producer_request_hold", + quotaType: "destination_storage", + }, + retentionForm: { + retentionTimeInMinutes: -1, + retentionSizeInMb: -1, + }, + }; +} + +export function policyFormsFromEffectivePolicies(raw: unknown, fallback: MqPolicyForms): MqPolicyForms { + const root = objectRecord(raw); + const topicPolicies = objectRecord(root.topicPolicies); + const namespacePolicies = objectRecord(root.namespacePolicies); + const namespaceSource = Object.keys(namespacePolicies).length ? namespacePolicies : root; + + const publish = firstRecord(topicPolicies.publishRate, root.publishRate, namespaceSource.publishRate, namespaceSource.publishMaxMessageRate); + const dispatch = firstRecord(topicPolicies.dispatchRate, root.dispatchRate, namespaceSource.dispatchRate); + const subscribe = firstRecord(topicPolicies.subscribeRate, root.subscribeRate, namespaceSource.subscribeRate); + const backlog = normalizeBacklogQuota(firstDefined(topicPolicies.backlogQuota, root.backlogQuota, namespaceSource.backlogQuota, namespaceSource.backlogQuotaMap, namespaceSource.backlog_quota_map)); + const retention = firstRecord(topicPolicies.retention, root.retention, namespaceSource.retention, namespaceSource.retentionPolicies, namespaceSource.retention_policies); + + return { + publishForm: publish + ? { + publishThrottlingRateInMsg: numberField(publish.publishThrottlingRateInMsg) ?? fallback.publishForm.publishThrottlingRateInMsg, + publishThrottlingRateInByte: numberField(publish.publishThrottlingRateInByte) ?? fallback.publishForm.publishThrottlingRateInByte, + } + : { ...fallback.publishForm }, + dispatchForm: dispatch + ? { + dispatchThrottlingRateInMsg: numberField(dispatch.dispatchThrottlingRateInMsg) ?? fallback.dispatchForm.dispatchThrottlingRateInMsg, + dispatchThrottlingRateInByte: numberField(dispatch.dispatchThrottlingRateInByte) ?? fallback.dispatchForm.dispatchThrottlingRateInByte, + ratePeriodInSecond: numberField(dispatch.ratePeriodInSecond) ?? fallback.dispatchForm.ratePeriodInSecond, + } + : { ...fallback.dispatchForm }, + subscribeForm: subscribe + ? { + subscribeThrottlingRatePerConsumer: numberField(subscribe.subscribeThrottlingRatePerConsumer) ?? fallback.subscribeForm.subscribeThrottlingRatePerConsumer, + ratePeriodInSecond: numberField(subscribe.ratePeriodInSecond) ?? fallback.subscribeForm.ratePeriodInSecond, + } + : { ...fallback.subscribeForm }, + backlogForm: backlog + ? { + limitSize: numberField(backlog.limitSize) ?? numberField(backlog.limit) ?? fallback.backlogForm.limitSize, + limitTime: numberField(backlog.limitTime) ?? fallback.backlogForm.limitTime, + policy: stringField(backlog.policy) || fallback.backlogForm.policy, + quotaType: stringField(backlog.quotaType) || fallback.backlogForm.quotaType, + } + : { ...fallback.backlogForm }, + retentionForm: retention + ? { + retentionTimeInMinutes: numberField(retention.retentionTimeInMinutes) ?? fallback.retentionForm.retentionTimeInMinutes, + retentionSizeInMb: numberField(retention.retentionSizeInMb) ?? numberField(retention.retentionSizeInMB) ?? fallback.retentionForm.retentionSizeInMb, + } + : { ...fallback.retentionForm }, + }; +} + +function normalizeBacklogQuota(value: unknown): Record | undefined { + const quota = objectRecord(value); + if (!Object.keys(quota).length) return undefined; + if (hasQuotaFields(quota)) return quota; + + const entries = Object.entries(quota); + for (const [quotaType, body] of entries) { + const normalized = objectRecord(body); + if (hasQuotaFields(normalized)) { + return { + ...normalized, + quotaType: normalized.quotaType ?? quotaType, + }; + } + } + return undefined; +} + +function hasQuotaFields(value: Record): boolean { + return "limit" in value || "limitSize" in value || "limitTime" in value || "policy" in value; +} + +function firstRecord(...values: unknown[]): Record | undefined { + for (const value of values) { + const record = objectRecord(value); + if (Object.keys(record).length) return record; + } + return undefined; +} + +function firstDefined(...values: unknown[]) { + return values.find((value) => value !== undefined && value !== null); +} + +function objectRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function numberField(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim() !== "") { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : undefined; + } + return undefined; +} + +function stringField(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} diff --git a/apps/desktop/src/lib/mqTenantForm.ts b/apps/desktop/src/lib/mqTenantForm.ts new file mode 100644 index 000000000..f7a6edef8 --- /dev/null +++ b/apps/desktop/src/lib/mqTenantForm.ts @@ -0,0 +1,42 @@ +import type { TenantConfig } from "@/types/mq"; + +export function normalizeClusterOptions(clusters: readonly unknown[]): string[] { + const seen = new Set(); + const result: string[] = []; + + for (const cluster of clusters) { + if (typeof cluster !== "string") continue; + const name = cluster.trim(); + if (!name || seen.has(name)) continue; + seen.add(name); + result.push(name); + } + + return result; +} + +export function mqClusterOptionsFromExtra(extra: unknown): string[] { + if (!extra || typeof extra !== "object") return []; + + const clusters = (extra as { clusters?: unknown }).clusters; + if (!Array.isArray(clusters)) return []; + + return normalizeClusterOptions(clusters); +} + +export function defaultTenantConfig(clusterOptions: readonly string[]): TenantConfig { + return { + adminRoles: [], + allowedClusters: normalizeClusterOptions(clusterOptions), + }; +} + +export function validateTenantForm(name: string | undefined, config: TenantConfig): string | undefined { + if (!name?.trim()) { + return "Tenant name is required"; + } + if (!normalizeClusterOptions(config.allowedClusters).length) { + return "Allowed clusters are required"; + } + return undefined; +} diff --git a/apps/desktop/src/lib/mqTokenErrors.ts b/apps/desktop/src/lib/mqTokenErrors.ts new file mode 100644 index 000000000..bdc79fa7d --- /dev/null +++ b/apps/desktop/src/lib/mqTokenErrors.ts @@ -0,0 +1,31 @@ +export type MqTokenIssueErrorView = + | { + kind: "missingSigningKey"; + title: string; + message: string; + detail: string; + } + | { + kind: "generic"; + title?: undefined; + message: string; + detail?: undefined; + }; + +export function formatMqTokenIssueError(error: unknown): MqTokenIssueErrorView { + const message = error instanceof Error ? error.message : String(error ?? ""); + if (isMissingTokenSigningConfig(message)) { + return { + kind: "missingSigningKey", + title: "未配置 Token 签发密钥", + message: "当前 MQ 连接还没有配置 Broker Token 签发密钥,无法生成客户端 Token。", + detail: "请编辑该连接,在 MQ 配置中设置“Broker Token 签发”为 HS256 SECRET 或 RS256 PRIVATE,并填写签发密钥后再生成。", + }; + } + return { kind: "generic", message }; +} + +function isMissingTokenSigningConfig(message: string) { + const normalized = message.toLowerCase(); + return normalized.includes("token signing is not configured") || normalized.includes("token signing key is required") || normalized.includes("broker token signing key is required"); +} diff --git a/apps/desktop/src/lib/openTabsPersistence.ts b/apps/desktop/src/lib/openTabsPersistence.ts index a7aaa7d86..000b78320 100644 --- a/apps/desktop/src/lib/openTabsPersistence.ts +++ b/apps/desktop/src/lib/openTabsPersistence.ts @@ -32,6 +32,7 @@ export interface SavedOpenTab { whereInput?: string; pinned?: boolean; mode?: QueryTab["mode"]; + mqTenant?: string; structureTableName?: string; objectBrowser?: QueryTab["objectBrowser"]; objectSource?: QueryTab["objectSource"]; @@ -69,6 +70,7 @@ export function serializeOpenTabs(tabs: QueryTab[]): SavedOpenTab[] { ...(tab.whereInput !== undefined ? { whereInput: tab.whereInput } : {}), pinned: tab.pinned, mode: tab.mode, + ...(tab.mqTenant !== undefined ? { mqTenant: tab.mqTenant } : {}), ...(tab.structureTableName !== undefined ? { structureTableName: tab.structureTableName } : {}), objectBrowser: tab.objectBrowser, objectSource: tab.objectSource, diff --git a/apps/desktop/src/lib/sidebarActiveTabTarget.ts b/apps/desktop/src/lib/sidebarActiveTabTarget.ts index b40d16bb7..c12a83c17 100644 --- a/apps/desktop/src/lib/sidebarActiveTabTarget.ts +++ b/apps/desktop/src/lib/sidebarActiveTabTarget.ts @@ -19,6 +19,11 @@ export type ActiveTabSidebarTarget = type: "etcd-root"; connectionId: string; } + | { + type: "mq-tenant"; + connectionId: string; + tenant: string; + } | { type: "query-context"; connectionId: string; @@ -60,6 +65,10 @@ export function activeTabSidebarTarget(tab: QueryTab | undefined | null): Active return { type: "etcd-root", connectionId: tab.connectionId }; } + if (tab.mode === "mq" && tab.mqTenant) { + return { type: "mq-tenant", connectionId: tab.connectionId, tenant: tab.mqTenant }; + } + if (tab.savedSqlId) { return { type: "saved-sql-file", savedSqlId: tab.savedSqlId }; } @@ -101,6 +110,10 @@ export function matchesTarget(node: TreeNode, target: ActiveTabSidebarTarget): b return node.type === "etcd-root" && node.connectionId === target.connectionId; } + if (target.type === "mq-tenant") { + return node.type === "mq-tenant" && node.connectionId === target.connectionId && (node.mqTenant || node.label) === target.tenant; + } + if (target.type === "saved-sql-file") { return node.type === "saved-sql-file" && node.savedSqlId === target.savedSqlId; } diff --git a/apps/desktop/src/lib/sidebarTreeItemLayout.ts b/apps/desktop/src/lib/sidebarTreeItemLayout.ts index 4d758d7d5..cf9ee9e88 100644 --- a/apps/desktop/src/lib/sidebarTreeItemLayout.ts +++ b/apps/desktop/src/lib/sidebarTreeItemLayout.ts @@ -1,6 +1,6 @@ import type { TreeNodeType } from "@/types/database"; -const leafTypes: Set = new Set(["column", "index", "fkey", "trigger", "procedure", "function", "package", "package-body", "object-browser", "redis-db", "mongo-collection", "elasticsearch-index", "user-admin", "saved-sql-file", "load-more"]); +const leafTypes: Set = new Set(["column", "index", "fkey", "trigger", "procedure", "function", "package", "package-body", "object-browser", "redis-db", "mq-tenant", "mongo-collection", "elasticsearch-index", "user-admin", "saved-sql-file", "load-more"]); const fullWidthLabelTypes: Set = new Set(["table", "view", "materialized_view", "mongo-collection", "elasticsearch-index"]); diff --git a/apps/desktop/src/lib/tauri.ts b/apps/desktop/src/lib/tauri.ts index f2ea33445..f4ba30d45 100644 --- a/apps/desktop/src/lib/tauri.ts +++ b/apps/desktop/src/lib/tauri.ts @@ -1721,3 +1721,5 @@ export async function exportQueryResultMarkdown(filePath: string, columns: strin }, }); } + +export * from "./mq-tauri"; diff --git a/apps/desktop/src/lib/treeNodeClick.ts b/apps/desktop/src/lib/treeNodeClick.ts index 1f8644d22..4e4639caf 100644 --- a/apps/desktop/src/lib/treeNodeClick.ts +++ b/apps/desktop/src/lib/treeNodeClick.ts @@ -7,7 +7,7 @@ export type SidebarSelectionCopyAction = "copy-name" | "none"; export type SidebarActivation = "single" | "double"; const dataNodeTypes = new Set(["table", "view", "materialized_view"]); -const toggleLeafNodeTypes = new Set(["redis-db", "mongo-collection", "elasticsearch-index", "user-admin"]); +const toggleLeafNodeTypes = new Set(["redis-db", "mq-tenant", "mongo-collection", "elasticsearch-index", "user-admin"]); const objectBrowserNodeTypes = new Set(["database", "schema", "object-browser"]); const sourceNodeTypes = new Set(["materialized_view", "procedure", "function", "sequence", "package", "package-body"]); const savedSqlNodeTypes = new Set(["saved-sql-file"]); diff --git a/apps/desktop/src/stores/__tests__/connectionStore.mq.spec.ts b/apps/desktop/src/stores/__tests__/connectionStore.mq.spec.ts new file mode 100644 index 000000000..22d5853e2 --- /dev/null +++ b/apps/desktop/src/stores/__tests__/connectionStore.mq.spec.ts @@ -0,0 +1,113 @@ +import { createPinia, setActivePinia } from "pinia"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { ConnectionConfig, TreeNode } from "@/types/database"; + +function installLocalStorage() { + const data = new Map(); + vi.stubGlobal("localStorage", { + getItem: vi.fn((key: string) => data.get(key) ?? null), + setItem: vi.fn((key: string, value: string) => data.set(key, value)), + removeItem: vi.fn((key: string) => data.delete(key)), + }); +} + +function mqConnection(): ConnectionConfig { + return { + id: "mq-1", + name: "Apache Pulsar", + db_type: "mq", + host: "127.0.0.1", + port: 8080, + user: "", + password: "", + database: "", + readonly: false, + read_only: false, + ssl_mode: "disabled", + color: "#888", + external_config: { + systemKind: "pulsar", + adminUrl: "http://127.0.0.1:8080", + auth: { kind: "none" }, + }, + } as ConnectionConfig; +} + +describe("connectionStore MQ sidebar tree", () => { + beforeEach(() => { + vi.resetModules(); + vi.unstubAllGlobals(); + installLocalStorage(); + setActivePinia(createPinia()); + }); + + it("loads Pulsar tenants under a message queue connection", async () => { + const mqListTenants = vi.fn().mockResolvedValue([ + { name: "public", adminRoles: [], allowedClusters: [] }, + { name: "tenant-a", adminRoles: [], allowedClusters: [] }, + ]); + + vi.doMock("@/lib/tauriRuntime", () => ({ isTauriRuntime: () => false })); + vi.doMock("@/lib/api", () => ({ + deleteSchemaCachePrefix: vi.fn().mockResolvedValue(undefined), + listDatabases: vi.fn().mockResolvedValue([]), + loadSchemaCache: vi.fn().mockResolvedValue(null), + mqListTenants, + saveSchemaCache: vi.fn().mockResolvedValue(undefined), + })); + + const { useConnectionStore } = await import("@/stores/connectionStore"); + const store = useConnectionStore(); + const connection = mqConnection(); + const node: TreeNode = { + id: connection.id, + label: connection.name, + type: "connection", + connectionId: connection.id, + isExpanded: false, + children: [], + }; + + store.connections = [connection]; + store.connectedIds.add(connection.id); + store.treeNodes = [node]; + + await store.refreshTreeNode(node); + + expect(mqListTenants).toHaveBeenCalledWith(connection.id); + expect(node.children?.map((child) => ({ id: child.id, label: child.label, type: child.type, tenant: (child as TreeNode & { mqTenant?: string }).mqTenant }))).toEqual([ + { id: "mq-1:mq-tenant:public", label: "public", type: "mq-tenant", tenant: "public" }, + { id: "mq-1:mq-tenant:tenant-a", label: "tenant-a", type: "mq-tenant", tenant: "tenant-a" }, + ]); + expect(node.isExpanded).toBe(true); + }); + + it("stores the selected tenant when opening an MQ admin tab", async () => { + const { useConnectionStore } = await import("@/stores/connectionStore"); + const { useQueryStore } = await import("@/stores/queryStore"); + const connectionStore = useConnectionStore(); + const queryStore = useQueryStore(); + connectionStore.connections = [mqConnection()]; + + const firstTabId = queryStore.openMqAdmin("mq-1", { tenant: "public" }); + expect(queryStore.tabs.find((tab) => tab.id === firstTabId)?.mqTenant).toBe("public"); + + const secondTabId = queryStore.openMqAdmin("mq-1", { tenant: "tenant-a" }); + expect(secondTabId).toBe(firstTabId); + expect(queryStore.tabs.find((tab) => tab.id === firstTabId)?.mqTenant).toBe("tenant-a"); + }); + + it("preserves the selected tenant when duplicating an MQ admin tab", async () => { + const { useConnectionStore } = await import("@/stores/connectionStore"); + const { useQueryStore } = await import("@/stores/queryStore"); + const connectionStore = useConnectionStore(); + const queryStore = useQueryStore(); + connectionStore.connections = [mqConnection()]; + + const tabId = queryStore.openMqAdmin("mq-1", { tenant: "public" }); + queryStore.duplicateTab(tabId); + + expect(queryStore.tabs).toHaveLength(2); + expect(queryStore.tabs[1]?.mqTenant).toBe("public"); + }); +}); diff --git a/apps/desktop/src/stores/connectionStore.ts b/apps/desktop/src/stores/connectionStore.ts index cd7685891..4dfbd01e8 100644 --- a/apps/desktop/src/stores/connectionStore.ts +++ b/apps/desktop/src/stores/connectionStore.ts @@ -731,6 +731,8 @@ export const useConnectionStore = defineStore("connection", () => { await loadMongoDatabases(connectionId); } else if (config.db_type === "elasticsearch") { await loadElasticsearchIndices(connectionId); + } else if (config.db_type === "mq") { + await loadMqTenants(connectionId, { force: true }); } else { await loadDatabases(connectionId, { force: true }); } @@ -1016,6 +1018,36 @@ export const useConnectionStore = defineStore("connection", () => { } } + async function loadMqTenants(connectionId: string, options?: LoadTreeOptions) { + const node = findNode(treeNodes.value, connectionId); + if (!node) return; + + node.isLoading = true; + try { + await ensureConnected(connectionId); + if (useCachedChildren(node, options)) return; + + const tenants = await api.mqListTenants(connectionId); + const tenantNames = sortSidebarNames(tenants.map((tenant) => tenant.name).filter((name) => !!name.trim())); + setChildren( + node, + tenantNames.map((tenant) => ({ + id: schemaCacheKey(connectionId, "mq-tenant", tenant), + label: tenant, + type: "mq-tenant" as const, + connectionId, + mqTenant: tenant, + })), + ); + node.isExpanded = true; + } catch (e) { + recordMetadataLoadError(connectionId, e); + throw e; + } finally { + node.isLoading = false; + } + } + function updateRedisDbKeyStats(connectionId: string, db: number, stats: { loaded?: number; total?: number; totalDelta?: number }) { const node = findNode(treeNodes.value, `${connectionId}:db${db}`); if (!node || node.type !== "redis-db") return; @@ -1588,6 +1620,8 @@ export const useConnectionStore = defineStore("connection", () => { await loadMongoDatabases(node.connectionId); } else if (config?.db_type === "elasticsearch") { await loadElasticsearchIndices(node.connectionId); + } else if (config?.db_type === "mq") { + await loadMqTenants(node.connectionId, options); } else { await loadDatabases(node.connectionId, options); } @@ -2697,6 +2731,7 @@ export const useConnectionStore = defineStore("connection", () => { loadDatabases, loadRedisDatabases, loadEtcdRoot, + loadMqTenants, updateRedisDbKeyStats, loadMongoDatabases, loadElasticsearchIndices, diff --git a/apps/desktop/src/stores/queryStore.ts b/apps/desktop/src/stores/queryStore.ts index acf27c31b..65408ebde 100644 --- a/apps/desktop/src/stores/queryStore.ts +++ b/apps/desktop/src/stores/queryStore.ts @@ -479,6 +479,33 @@ export const useQueryStore = defineStore("query", () => { return id; } + function openMqAdmin(connectionId: string, target?: { tenant?: string }) { + const existing = tabs.value.find((tab) => tab.mode === "mq" && tab.connectionId === connectionId); + if (existing) { + if (target?.tenant) existing.mqTenant = target.tenant; + activeTabId.value = existing.id; + return existing.id; + } + + const conn = useConnectionStore().getConfig(connectionId); + const id = uuid(); + const tab: QueryTab = { + id, + title: `${conn?.name || "Message Queue"} Admin`, + connectionId, + database: conn?.database || "", + sql: "", + isExecuting: false, + isCancelling: false, + isExplaining: false, + mode: "mq", + mqTenant: target?.tenant, + }; + tabs.value.push(tab); + activeTabId.value = id; + return id; + } + function openTableStructure(connectionId: string, database: string, schema?: string, tableName?: string) { const resolvedTableName = tableName || ""; if (resolvedTableName) { @@ -641,6 +668,7 @@ export const useQueryStore = defineStore("query", () => { isExplaining: false, explainExecutionId: undefined, mode: original.mode, + mqTenant: original.mqTenant, structureTableName: original.structureTableName, objectBrowser: original.objectBrowser ? { ...original.objectBrowser } : undefined, objectSource: original.objectSource ? { ...original.objectSource } : undefined, @@ -1948,6 +1976,7 @@ export const useQueryStore = defineStore("query", () => { renameTab, openObjectBrowser, openUserAdmin, + openMqAdmin, openTableStructure, linkSavedSql, openSavedSql, diff --git a/apps/desktop/src/types/database.ts b/apps/desktop/src/types/database.ts index 9678cf243..fcf7f6c6b 100644 --- a/apps/desktop/src/types/database.ts +++ b/apps/desktop/src/types/database.ts @@ -52,7 +52,8 @@ export type DatabaseType = | "etcd" | "iris" | "influxdb" - | "jdbc"; + | "jdbc" + | "mq"; export interface SqlSnippet { id: string; @@ -100,6 +101,7 @@ export interface ConnectionConfig { redis_key_separator?: string; etcd_endpoints?: string; gbase_server?: string; + external_config?: unknown; one_time?: boolean; read_only?: boolean; } @@ -419,6 +421,7 @@ export type TreeNodeType = | "fkey" | "trigger" | "redis-db" + | "mq-tenant" | "etcd-root" | "mongo-db" | "mongo-collection" @@ -447,6 +450,7 @@ export interface TreeNode { pinned?: boolean; connectionId?: string; database?: string; + mqTenant?: string; schema?: string; tableName?: string; comment?: string | null; @@ -517,7 +521,8 @@ export interface QueryTab { executionId?: string; isExplaining?: boolean; explainExecutionId?: string; - mode: "data" | "query" | "redis" | "mongo" | "etcd" | "objects" | "structure" | "users"; + mode: "data" | "query" | "redis" | "mongo" | "etcd" | "mq" | "objects" | "structure" | "users"; + mqTenant?: string; structureTableName?: string; objectBrowser?: { schema?: string; diff --git a/apps/desktop/src/types/mq.ts b/apps/desktop/src/types/mq.ts new file mode 100644 index 000000000..0e2c26ddc --- /dev/null +++ b/apps/desktop/src/types/mq.ts @@ -0,0 +1,251 @@ +// Message queue admin types, matching dbx-core/src/mq/types.rs + +export type MqSystemKind = "pulsar" | "kafka" | "rocketmq"; + +export interface MqCapabilities { + supportsTenants: boolean; + supportsNamespaces: boolean; + supportsPartitionedTopics: boolean; + supportsSubscriptions: boolean; + supportsCreateSubscription: boolean; + supportsResetCursor: boolean; + supportsSkipMessages: boolean; + supportsClearBacklog: boolean; + supportsPeekMessages: boolean; + supportsExpireMessages: boolean; + supportsRateLimits: boolean; + supportsBacklogQuota: boolean; + supportsRetention: boolean; + supportsPermissions: boolean; + supportsGeoReplication: boolean; + supportsTokenManagement: boolean; + supportsRawAdminApi: boolean; +} + +export interface MqClusterInfo { + systemKind: MqSystemKind; + serverVersion?: string; + resolvedProfile: string; + versionDetection: string; + capabilities: MqCapabilities; + extra?: unknown; +} + +export interface MqAuth { + kind: "none" | "token" | "basic" | "apiKey" | "oauth2"; + token?: string; + username?: string; + password?: string; + header?: string; + value?: string; + issuerUrl?: string; + clientId?: string; + clientSecret?: string; + audience?: string; + scope?: string; +} + +export interface MqAdminConfig { + systemKind: MqSystemKind; + adminUrl: string; + auth?: MqAuth; + tlsSkipVerify?: boolean; + pinnedVersion?: string; + tokenSigning?: MqTokenSigningConfig; + extra?: unknown; +} + +export type MqTokenSigningAlgorithm = "hs256" | "rs256"; + +export interface MqTokenSigningConfig { + algorithm: MqTokenSigningAlgorithm; + key: string; +} + +export interface MqTokenIssueRequest { + subject: string; + expiresInSeconds?: number; + scope?: PolicyScope; + actions: AuthAction[]; + note?: string; +} + +export interface MqTokenRecord { + id: string; + connectionId: string; + subject: string; + algorithm: MqTokenSigningAlgorithm; + tokenFingerprint: string; + scope?: PolicyScope; + actions: AuthAction[]; + expiresAt?: string; + createdAt: string; + note: string; +} + +export interface MqIssuedToken { + token: string; + record: MqTokenRecord; +} + +// Tenant +export interface TenantInfo { + name: string; + adminRoles: string[]; + allowedClusters: string[]; +} + +export interface TenantConfig { + adminRoles: string[]; + allowedClusters: string[]; +} + +// Namespace +export interface NamespaceRef { + tenant: string; + namespace: string; +} + +export interface NamespaceInfo { + tenant: string; + namespace: string; + adminRoles: string[]; +} + +export interface NamespaceConfig { + clusters?: string[]; + bundles?: number; +} + +// Topic +export interface TopicRef { + tenant: string; + namespace: string; + topic: string; + persistent: boolean; + partitioned?: boolean; +} + +export interface TopicInfo { + name: string; + shortName: string; + partitioned: boolean; + partitions?: number; + persistent: boolean; +} + +export interface ListTopicsOpts { + includeNonPersistent?: boolean; +} + +export interface TopicStats { + msgRateIn: number; + msgRateOut: number; + msgThroughputIn: number; + msgThroughputOut: number; + storageSize: number; + backlogSize: number; + msgInCounter: number; + msgOutCounter: number; + subscriptionCount: number; + producerCount: number; + raw?: unknown; +} + +// Subscription +export interface SubscriptionInfo { + name: string; + subType: string; + msgBacklog: number; + msgRateOut: number; + msgThroughputOut: number; + consumers: ConsumerInfo[]; +} + +export interface ConsumerInfo { + consumerName: string; + msgRateOut: number; + msgThroughputOut: number; + availablePermits: number; + address: string; + clientVersion: string; +} + +export interface ProducerInfo { + producerId: number; + producerName: string; + msgRateIn: number; + msgThroughputIn: number; + address: string; + clientVersion: string; +} + +export type ResetPosition = { kind: "earliest" } | { kind: "latest" } | { kind: "timestamp"; timestampMs: number } | { kind: "messageId"; ledgerId: number; entryId: number }; + +export type SkipCount = { kind: "all" } | { kind: "count"; count: number }; + +export interface BacklogStats { + msgBacklog: number; + backlogSize: number; +} + +export interface PeekedMessage { + position: number; + messageId?: string; + key?: string; + publishTime?: string; + eventTime?: string; + properties: Record; + headers: Record; + payloadBase64: string; + payloadText?: string; +} + +// Policy scope +export type PolicyScope = { level: "namespace"; tenant: string; namespace: string } | { level: "topic"; tenant: string; namespace: string; topic: string; persistent: boolean }; + +export interface PublishRate { + publishThrottlingRateInMsg: number; + publishThrottlingRateInByte: number; +} + +export interface DispatchRate { + dispatchThrottlingRateInMsg: number; + dispatchThrottlingRateInByte: number; + ratePeriodInSecond: number; +} + +export interface SubscribeRate { + subscribeThrottlingRatePerConsumer: number; + ratePeriodInSecond: number; +} + +export interface BacklogQuota { + limitSize: number; + limitTime: number; + policy: string; + quotaType: string; +} + +export interface RetentionPolicy { + retentionTimeInMinutes: number; + retentionSizeInMb: number; +} + +export type AuthAction = "produce" | "consume" | "functions" | "sources" | "sinks" | "packages"; + +export type PermissionMap = Record; + +// Raw request +export interface MqRawRequest { + method: string; + path: string; + query?: Record; + body?: unknown; +} + +export interface MqRawResponse { + status: number; + body: unknown; + text?: string; +} diff --git a/crates/dbx-core/Cargo.toml b/crates/dbx-core/Cargo.toml index 505b7184c..7d7e914a5 100644 --- a/crates/dbx-core/Cargo.toml +++ b/crates/dbx-core/Cargo.toml @@ -5,8 +5,9 @@ edition = "2021" license = "Apache-2.0" [features] -default = ["duckdb-bundled"] +default = ["duckdb-bundled", "mq-admin"] duckdb-bundled = ["duckdb/bundled"] +mq-admin = [] [dependencies] serde = { version = "1.0", features = ["derive"] } @@ -44,6 +45,7 @@ portpicker = "0.1.1" csv = "1" calamine = "0.30.1" base64 = "0.22" +jsonwebtoken = "9" aes-gcm = "0.10" argon2 = "0.5" sha2 = "0.10" diff --git a/crates/dbx-core/assets/database-drivers.manifest.json b/crates/dbx-core/assets/database-drivers.manifest.json index 324d585e9..a134f6496 100644 --- a/crates/dbx-core/assets/database-drivers.manifest.json +++ b/crates/dbx-core/assets/database-drivers.manifest.json @@ -1462,6 +1462,35 @@ "driverManagement": true } }, + { + "dbType": "mq", + "label": "Message Queue", + "runtimeMode": "native", + "mcpMode": "unsupported", + "singleConnectionPool": false, + "metadataConnectionScoped": false, + "skipTcpProbe": true, + "defaultPort": 8080, + "supportLevel": "connect", + "capabilities": { + "queryExecution": false, + "metadataBrowse": false, + "objectBrowser": false, + "objectSource": false, + "schemaSearch": false, + "diagram": false, + "tableDataEdit": false, + "tableStructureEdit": false, + "tableImport": false, + "dataTransfer": false, + "sqlFileExecution": false, + "databaseCreate": false, + "fieldLineage": false, + "sqlExplain": false, + "userAdmin": false, + "driverManagement": false + } + }, { "dbType": "iotdb", "label": "Apache IoTDB", diff --git a/crates/dbx-core/src/cloud_sync.rs b/crates/dbx-core/src/cloud_sync.rs index 19f1679ef..257d4bab1 100644 --- a/crates/dbx-core/src/cloud_sync.rs +++ b/crates/dbx-core/src/cloud_sync.rs @@ -10,6 +10,9 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use crate::ai::AiConfig; +use crate::connection_secrets::{ + MQ_AUTH_API_KEY_VALUE_KEY, MQ_AUTH_CLIENT_SECRET_KEY, MQ_AUTH_PASSWORD_KEY, MQ_AUTH_TOKEN_KEY, MQ_TOKEN_SIGNING_KEY, +}; use crate::models::connection::{ConnectionConfig, TransportLayerConfig}; use crate::saved_sql::SavedSqlLibrary; use crate::storage::{DesktopSettings, Storage}; @@ -23,6 +26,11 @@ const SECRET_KEYS: &[&str] = &[ "proxy_password", "redis_sentinel_password", "connection_string", + MQ_AUTH_TOKEN_KEY, + MQ_AUTH_PASSWORD_KEY, + MQ_AUTH_API_KEY_VALUE_KEY, + MQ_AUTH_CLIENT_SECRET_KEY, + MQ_TOKEN_SIGNING_KEY, ]; const SSH_TUNNEL_SECRET_PREFIX: &str = "ssh_tunnels."; const TRANSPORT_LAYER_SECRET_PREFIX: &str = "transport_layers."; @@ -362,11 +370,44 @@ async fn build_sensitive_payload( if let Some(connection_string) = &config.connection_string { push_secret(&mut connection_secrets, &config.id, "connection_string", connection_string); } + push_mq_external_config_secrets(&mut connection_secrets, config); } Ok(SensitiveSyncPayload { connection_secrets, ai_config: storage.load_ai_config().await? }) } +fn push_mq_external_config_secrets(secrets: &mut Vec, config: &ConnectionConfig) { + let Some(external_config) = config.external_config.as_ref() else { + return; + }; + if let Some(auth) = external_config.get("auth").and_then(serde_json::Value::as_object) { + match auth.get("kind").and_then(serde_json::Value::as_str) { + Some("token") => push_json_secret(secrets, &config.id, MQ_AUTH_TOKEN_KEY, auth, "token"), + Some("basic") => push_json_secret(secrets, &config.id, MQ_AUTH_PASSWORD_KEY, auth, "password"), + Some("apiKey") | Some("api_key") | Some("apikey") => { + push_json_secret(secrets, &config.id, MQ_AUTH_API_KEY_VALUE_KEY, auth, "value") + } + Some("oauth2") => push_json_secret(secrets, &config.id, MQ_AUTH_CLIENT_SECRET_KEY, auth, "clientSecret"), + _ => {} + } + } + if let Some(signing) = external_config.get("tokenSigning").and_then(serde_json::Value::as_object) { + push_json_secret(secrets, &config.id, MQ_TOKEN_SIGNING_KEY, signing, "key"); + } +} + +fn push_json_secret( + secrets: &mut Vec, + connection_id: &str, + key: &str, + object: &serde_json::Map, + field: &str, +) { + if let Some(secret) = object.get(field).and_then(serde_json::Value::as_str) { + push_secret(secrets, connection_id, key, secret); + } +} + fn push_secret(secrets: &mut Vec, connection_id: &str, key: &str, secret: &str) { if secret.is_empty() { return; diff --git a/crates/dbx-core/src/connection.rs b/crates/dbx-core/src/connection.rs index e45d6eb1b..3e1789618 100644 --- a/crates/dbx-core/src/connection.rs +++ b/crates/dbx-core/src/connection.rs @@ -70,7 +70,14 @@ pub enum PoolKind { InfluxDb(db::influxdb_driver::InfluxdbClient), Agent(Arc>), ExternalTabular(ExternalTabularHandle), - ExternalDriver { driver_id: String, config: Arc, session: Arc }, + ExternalDriver { + driver_id: String, + config: Arc, + session: Arc, + }, + /// Message queue admin connection (not a data query pool; serves as a + /// marker that this connection_id is a valid MQ admin connection). + MessageQueue, } pub struct AppState { @@ -83,6 +90,8 @@ pub struct AppState { pub storage: Storage, pub plugins: PluginRegistry, pub agent_manager: crate::agent_manager::AgentManager, + #[cfg(feature = "mq-admin")] + pub mq_registry: crate::mq::MqAdminRegistry, } pub fn metadata_connection_config(config: &ConnectionConfig) -> ConnectionConfig { @@ -263,6 +272,8 @@ impl AppState { agent_dir, app_version, ), + #[cfg(feature = "mq-admin")] + mq_registry: crate::mq::MqAdminRegistry::new(), } } @@ -743,6 +754,23 @@ impl AppState { } self.external_driver_pool("jdbc", &jdbc_config).await? } + #[cfg(feature = "mq-admin")] + DatabaseType::MessageQueue => { + // MQ admin connections don't hold a data query pool. We just test + // connectivity via the mq_registry and insert a marker so this + // connection_id is recognized as valid. + let mqc = self.mq_admin_config_for_connection(connection_id, &config).await?; + let adapter = self.mq_registry.build_transient_config(mqc).await?; + adapter.test_connection().await?; + PoolKind::MessageQueue + } + #[cfg(not(feature = "mq-admin"))] + DatabaseType::MessageQueue => { + return Err( + "Message queue admin support is not compiled in this build. Rebuild with the 'mq-admin' feature." + .to_string(), + ); + } }; self.insert_connection_pool(pool_key.clone(), pool, &db_config).await; @@ -773,6 +801,21 @@ impl AppState { Ok(("127.0.0.1".to_string(), local_port)) } + #[cfg(feature = "mq-admin")] + pub async fn mq_admin_config_for_connection( + &self, + connection_id: &str, + config: &ConnectionConfig, + ) -> Result { + let mqc = crate::mq::config::MqAdminConfig::from_connection(config)?; + if !config.has_effective_transport_layers() { + return Ok(mqc); + } + + let (host, port) = self.connection_host_port(connection_id, config).await?; + Ok(mqc.with_connect_override(&host, port)) + } + async fn remove_stale_connection_pool(&self, pool_key: &str) -> bool { let stale = { let connections = self.connections.read().await; @@ -1104,6 +1147,10 @@ impl AppState { removed.push((key, pool)); } } + drop(conns); + // Also drop the MQ admin adapter if this is an MQ connection. + #[cfg(feature = "mq-admin")] + self.mq_registry.drop_connection(connection_id).await; removed } @@ -1183,11 +1230,30 @@ fn connection_remote_endpoint(config: &ConnectionConfig) -> (String, u16) { .filter(|s| !s.is_empty()) .and_then(parse_jdbc_host_port) .unwrap_or_else(|| (config.host.clone(), config.port)) + } else if config.db_type == DatabaseType::MessageQueue { + parse_mq_admin_host_port(config).unwrap_or_else(|| (config.host.clone(), config.port)) } else { (config.host.clone(), config.port) } } +fn parse_mq_admin_host_port(config: &ConnectionConfig) -> Option<(String, u16)> { + let value = config + .external_config + .as_ref()? + .get("adminUrl") + .or_else(|| config.external_config.as_ref()?.get("admin_url"))? + .as_str()? + .trim(); + if value.is_empty() { + return None; + } + let url = reqwest::Url::parse(value).ok()?; + let host = url.host_str()?.to_string(); + let port = url.port_or_known_default()?; + Some((host, port)) +} + fn normalize_client_session_id(client_session_id: Option<&str>) -> Option { client_session_id.map(str::trim).filter(|session| !session.is_empty()).map(|session| session.replace(':', "_")) } @@ -1257,6 +1323,7 @@ pub async fn close_pool_kind(pool: PoolKind) { } PoolKind::ExternalTabular(_) => {} PoolKind::ExternalDriver { .. } => {} + PoolKind::MessageQueue => {} } } @@ -1513,9 +1580,9 @@ async fn detect_ob_oracle_mode(config: &ConnectionConfig, pool: &db::mysql::MySq #[cfg(test)] mod tests { use super::{ - agent_connect_timeout, connection_url_for_endpoint, database_connection_config, metadata_connection_config, - mysql_metadata_fallback_url, redacted_connection_url_for_endpoint, uses_bare_mysql_pool, uses_tcp_probe, - validate_h2_database_path, AppState, PoolKind, + agent_connect_timeout, connection_remote_endpoint, connection_url_for_endpoint, database_connection_config, + metadata_connection_config, mysql_metadata_fallback_url, redacted_connection_url_for_endpoint, + uses_bare_mysql_pool, uses_tcp_probe, validate_h2_database_path, AppState, PoolKind, }; use crate::agent_connection::{ agent_connect_params, mongo_legacy_error_with_auth_hint, mongo_uses_legacy_driver, @@ -2420,6 +2487,56 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + #[test] + fn mq_remote_endpoint_comes_from_admin_url_when_host_fields_are_empty() { + let mut config = mysql_config(None); + config.db_type = DatabaseType::MessageQueue; + config.host = String::new(); + config.port = 0; + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "https://broker.internal:8443/pulsar-admin?tenant=public", + "auth": { "kind": "none" } + })); + + assert_eq!(connection_remote_endpoint(&config), ("broker.internal".to_string(), 8443)); + } + + #[cfg(feature = "mq-admin")] + #[tokio::test] + async fn mq_admin_config_preserves_admin_url_and_uses_forwarded_connect_override() { + let (state, dir) = test_app_state().await; + let mut config = mysql_config(None); + config.id = "proxied-mq".to_string(); + config.db_type = DatabaseType::MessageQueue; + config.host = String::new(); + config.port = 0; + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "https://broker.internal:8443/pulsar-admin?tenant=public", + "auth": { "kind": "none" } + })); + config.transport_layers = vec![TransportLayerConfig::Proxy(ProxyTunnelConfig { + id: "proxy".to_string(), + name: String::new(), + enabled: true, + proxy_type: ProxyType::Socks5, + host: "127.0.0.1".to_string(), + port: 65000, + username: String::new(), + password: String::new(), + })]; + + let mqc = state.mq_admin_config_for_connection("proxied-mq", &config).await.unwrap(); + + assert_eq!(mqc.admin_url, "https://broker.internal:8443/pulsar-admin?tenant=public"); + let connect_override = mqc.connect_override.expect("MQ transport should set a connect override"); + assert_eq!(connect_override.host, "127.0.0.1"); + assert_ne!(connect_override.port, 8443); + state.proxy_tunnels.stop_tunnel("proxied-mq:transport:0").await; + let _ = std::fs::remove_dir_all(dir); + } + #[tokio::test] #[ignore = "requires a reachable GaussDB instance via environment variables"] async fn live_gaussdb_native_connection_succeeds() { diff --git a/crates/dbx-core/src/connection_secrets.rs b/crates/dbx-core/src/connection_secrets.rs index 8fa5af191..f0efbb5d5 100644 --- a/crates/dbx-core/src/connection_secrets.rs +++ b/crates/dbx-core/src/connection_secrets.rs @@ -1,4 +1,4 @@ -use crate::models::connection::{ConnectionConfig, TransportLayerConfig}; +use crate::models::connection::{ConnectionConfig, DatabaseType, TransportLayerConfig}; use std::collections::HashMap; use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -11,6 +11,13 @@ pub const TRANSPORT_LAYER_SECRET_PREFIX: &str = "transport_layers."; pub const PROXY_PASSWORD_KEY: &str = "proxy_password"; pub const REDIS_SENTINEL_PASSWORD_KEY: &str = "redis_sentinel_password"; pub const CONNECTION_STRING_KEY: &str = "connection_string"; +pub const MQ_AUTH_SECRET_PREFIX: &str = "mq.auth."; +pub const MQ_AUTH_TOKEN_KEY: &str = "mq.auth.token"; +pub const MQ_AUTH_PASSWORD_KEY: &str = "mq.auth.password"; +pub const MQ_AUTH_API_KEY_VALUE_KEY: &str = "mq.auth.api_key_value"; +pub const MQ_AUTH_CLIENT_SECRET_KEY: &str = "mq.auth.client_secret"; +pub const MQ_TOKEN_SIGNING_SECRET_PREFIX: &str = "mq.token_signing."; +pub const MQ_TOKEN_SIGNING_KEY: &str = "mq.token_signing.key"; pub trait ConnectionSecretStore { fn set_secret(&self, connection_id: &str, key: &str, secret: &str) -> Result<(), String>; @@ -31,7 +38,23 @@ impl FileSecretStore { } fn read_store(&self) -> HashMap { - std::fs::read_to_string(&self.path).ok().and_then(|json| serde_json::from_str(&json).ok()).unwrap_or_default() + match std::fs::read_to_string(&self.path) { + Ok(json) => match serde_json::from_str(&json) { + Ok(map) => map, + Err(e) => { + log::warn!( + "Failed to parse secret store at {:?}: {}. Returning empty store. This may indicate file corruption.", + self.path, e + ); + HashMap::default() + } + }, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => HashMap::default(), + Err(e) => { + log::warn!("Failed to read secret store at {:?}: {}. Returning empty store.", self.path, e); + HashMap::default() + } + } } fn write_store(&self, map: &HashMap) -> Result<(), String> { @@ -79,6 +102,8 @@ pub fn save_connections_to_file( } persist_secret(store, &config.id, REDIS_SENTINEL_PASSWORD_KEY, &config.redis_sentinel_password)?; persist_optional_secret(store, &config.id, CONNECTION_STRING_KEY, config.connection_string.as_deref())?; + persist_mq_auth_secrets(store, config)?; + persist_mq_token_signing_secret(store, config)?; // New configs persist transport-layer secrets only. Remove legacy transport secret slots after the // migrated layer values have been written so old configs do not keep two sources of truth. @@ -133,6 +158,8 @@ pub fn load_connections_from_file( } } } + hydrate_mq_auth_secrets(store, config, &mut needs_rewrite)?; + hydrate_mq_token_signing_secret(store, config, &mut needs_rewrite)?; } if needs_rewrite { @@ -294,6 +321,8 @@ fn delete_removed_connection_secrets( delete_secret_prefix(store, &config.id, SSH_TUNNEL_SECRET_PREFIX)?; delete_secret_prefix(store, &config.id, TRANSPORT_LAYER_SECRET_PREFIX)?; store.delete_secret(&config.id, CONNECTION_STRING_KEY)?; + delete_secret_prefix(store, &config.id, MQ_AUTH_SECRET_PREFIX)?; + delete_secret_prefix(store, &config.id, MQ_TOKEN_SIGNING_SECRET_PREFIX)?; } Ok(()) } @@ -331,6 +360,193 @@ fn delete_secret_prefix( store.delete_secret_prefix(connection_id, key_prefix) } +fn persist_mq_auth_secrets(store: &dyn ConnectionSecretStore, config: &ConnectionConfig) -> Result<(), String> { + if config.db_type != DatabaseType::MessageQueue { + delete_secret_prefix(store, &config.id, MQ_AUTH_SECRET_PREFIX)?; + return Ok(()); + } + + let Some(auth) = mq_auth_object(config.external_config.as_ref()) else { + delete_secret_prefix(store, &config.id, MQ_AUTH_SECRET_PREFIX)?; + return Ok(()); + }; + + match mq_auth_kind(auth).as_deref() { + Some("none") => delete_secret_prefix(store, &config.id, MQ_AUTH_SECRET_PREFIX)?, + Some("token") => replace_mq_auth_secret(store, &config.id, MQ_AUTH_TOKEN_KEY, auth, "token")?, + Some("basic") => replace_mq_auth_secret(store, &config.id, MQ_AUTH_PASSWORD_KEY, auth, "password")?, + Some("apiKey") | Some("api_key") | Some("apikey") => { + replace_mq_auth_secret(store, &config.id, MQ_AUTH_API_KEY_VALUE_KEY, auth, "value")? + } + Some("oauth2") => replace_mq_auth_secret(store, &config.id, MQ_AUTH_CLIENT_SECRET_KEY, auth, "clientSecret")?, + _ => delete_secret_prefix(store, &config.id, MQ_AUTH_SECRET_PREFIX)?, + } + + Ok(()) +} + +fn replace_mq_auth_secret( + store: &dyn ConnectionSecretStore, + connection_id: &str, + key: &str, + auth: &serde_json::Map, + field: &str, +) -> Result<(), String> { + let current = auth.get(field).and_then(serde_json::Value::as_str).filter(|secret| !secret.is_empty()); + let existing = if current.is_none() { store.get_secret(connection_id, key)? } else { None }; + delete_secret_prefix(store, connection_id, MQ_AUTH_SECRET_PREFIX)?; + match current { + Some(secret) => store.set_secret(connection_id, key, secret), + None => match existing { + Some(secret) => store.set_secret(connection_id, key, &secret), + None => Ok(()), + }, + } +} + +fn persist_mq_token_signing_secret(store: &dyn ConnectionSecretStore, config: &ConnectionConfig) -> Result<(), String> { + if config.db_type != DatabaseType::MessageQueue { + delete_secret_prefix(store, &config.id, MQ_TOKEN_SIGNING_SECRET_PREFIX)?; + return Ok(()); + } + + let Some(signing) = mq_token_signing_object(config.external_config.as_ref()) else { + delete_secret_prefix(store, &config.id, MQ_TOKEN_SIGNING_SECRET_PREFIX)?; + return Ok(()); + }; + + persist_json_secret_if_present(store, &config.id, MQ_TOKEN_SIGNING_KEY, signing, "key") +} + +fn hydrate_mq_auth_secrets( + store: &dyn ConnectionSecretStore, + config: &mut ConnectionConfig, + needs_rewrite: &mut bool, +) -> Result<(), String> { + if config.db_type != DatabaseType::MessageQueue { + return Ok(()); + } + + let Some(auth) = mq_auth_object_mut(config.external_config.as_mut()) else { + return Ok(()); + }; + + match mq_auth_kind(auth).as_deref() { + Some("token") => hydrate_json_secret(store, &config.id, MQ_AUTH_TOKEN_KEY, auth, "token", needs_rewrite)?, + Some("basic") => hydrate_json_secret(store, &config.id, MQ_AUTH_PASSWORD_KEY, auth, "password", needs_rewrite)?, + Some("apiKey") | Some("api_key") | Some("apikey") => { + hydrate_json_secret(store, &config.id, MQ_AUTH_API_KEY_VALUE_KEY, auth, "value", needs_rewrite)? + } + Some("oauth2") => { + hydrate_json_secret(store, &config.id, MQ_AUTH_CLIENT_SECRET_KEY, auth, "clientSecret", needs_rewrite)? + } + _ => {} + } + + Ok(()) +} + +fn hydrate_mq_token_signing_secret( + store: &dyn ConnectionSecretStore, + config: &mut ConnectionConfig, + needs_rewrite: &mut bool, +) -> Result<(), String> { + if config.db_type != DatabaseType::MessageQueue { + return Ok(()); + } + + let Some(signing) = mq_token_signing_object_mut(config.external_config.as_mut()) else { + return Ok(()); + }; + + hydrate_json_secret(store, &config.id, MQ_TOKEN_SIGNING_KEY, signing, "key", needs_rewrite) +} + +fn persist_json_secret_if_present( + store: &dyn ConnectionSecretStore, + connection_id: &str, + key: &str, + auth: &serde_json::Map, + field: &str, +) -> Result<(), String> { + match auth.get(field).and_then(serde_json::Value::as_str).filter(|secret| !secret.is_empty()) { + Some(secret) => store.set_secret(connection_id, key, secret), + None => Ok(()), + } +} + +fn hydrate_json_secret( + store: &dyn ConnectionSecretStore, + connection_id: &str, + key: &str, + auth: &mut serde_json::Map, + field: &str, + needs_rewrite: &mut bool, +) -> Result<(), String> { + match auth.get(field).and_then(serde_json::Value::as_str).filter(|secret| !secret.is_empty()) { + Some(secret) => { + store.set_secret(connection_id, key, secret)?; + *needs_rewrite = true; + } + None => { + if let Some(secret) = store.get_secret(connection_id, key)? { + auth.insert(field.to_string(), serde_json::Value::String(secret)); + } + } + } + Ok(()) +} + +fn scrub_mq_auth_secrets(config: &mut ConnectionConfig) { + let Some(auth) = mq_auth_object_mut(config.external_config.as_mut()) else { + return; + }; + match mq_auth_kind(auth).as_deref() { + Some("token") => scrub_json_secret(auth, "token"), + Some("basic") => scrub_json_secret(auth, "password"), + Some("apiKey") | Some("api_key") | Some("apikey") => scrub_json_secret(auth, "value"), + Some("oauth2") => scrub_json_secret(auth, "clientSecret"), + _ => {} + } +} + +fn scrub_mq_token_signing_secret(config: &mut ConnectionConfig) { + let Some(signing) = mq_token_signing_object_mut(config.external_config.as_mut()) else { + return; + }; + scrub_json_secret(signing, "key"); +} + +fn scrub_json_secret(auth: &mut serde_json::Map, field: &str) { + if auth.contains_key(field) { + auth.insert(field.to_string(), serde_json::Value::String(String::new())); + } +} + +fn mq_auth_kind(auth: &serde_json::Map) -> Option { + auth.get("kind").and_then(serde_json::Value::as_str).map(ToString::to_string) +} + +fn mq_auth_object(value: Option<&serde_json::Value>) -> Option<&serde_json::Map> { + value?.get("auth")?.as_object() +} + +fn mq_auth_object_mut( + value: Option<&mut serde_json::Value>, +) -> Option<&mut serde_json::Map> { + value?.get_mut("auth")?.as_object_mut() +} + +fn mq_token_signing_object(value: Option<&serde_json::Value>) -> Option<&serde_json::Map> { + value?.get("tokenSigning")?.as_object() +} + +fn mq_token_signing_object_mut( + value: Option<&mut serde_json::Value>, +) -> Option<&mut serde_json::Map> { + value?.get_mut("tokenSigning")?.as_object_mut() +} + fn ssh_tunnel_secret_segment(index: usize, hop: &crate::models::connection::SshTunnelConfig) -> String { if hop.id.trim().is_empty() { index.to_string() @@ -398,6 +614,8 @@ fn sanitize_connections(configs: &[ConnectionConfig]) -> Vec { } config.redis_sentinel_password.clear(); config.connection_string = None; + scrub_mq_auth_secrets(&mut config); + scrub_mq_token_signing_secret(&mut config); config }) .collect() @@ -411,7 +629,8 @@ pub fn secret_account(connection_id: &str, key: &str) -> String { mod tests { use super::{ load_connections_from_file, save_connections_to_file, ConnectionSecretStore, CONNECTION_STRING_KEY, - MAIN_PASSWORD_KEY, REDIS_SENTINEL_PASSWORD_KEY, SSH_PASSWORD_KEY, + MAIN_PASSWORD_KEY, MQ_AUTH_PASSWORD_KEY, MQ_AUTH_TOKEN_KEY, MQ_TOKEN_SIGNING_KEY, REDIS_SENTINEL_PASSWORD_KEY, + SSH_PASSWORD_KEY, }; use crate::models::connection::{ConnectionConfig, DatabaseType, SshTunnelConfig, TransportLayerConfig}; use std::cell::RefCell; @@ -669,4 +888,147 @@ mod tests { let loaded = load_connections_from_file(&path, &store).unwrap(); assert_eq!(loaded[0].connection_string.as_deref(), Some("mongodb://user:secret@localhost/app")); } + + #[test] + fn save_connections_moves_mq_auth_secrets_to_secret_store_and_restores_them() { + let path = temp_connections_file("mq-auth"); + let store = MemorySecretStore::default(); + let mut config = connection("pulsar", "", ""); + config.db_type = DatabaseType::MessageQueue; + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://localhost:8080", + "auth": { + "kind": "token", + "token": "mq-token-secret" + } + })); + + save_connections_to_file(&path, &[config], &store).unwrap(); + + assert_eq!(store.get_existing("pulsar", "mq.auth.token").as_deref(), Some("mq-token-secret")); + let persisted_json = std::fs::read_to_string(&path).unwrap(); + assert!(!persisted_json.contains("mq-token-secret")); + + let loaded = load_connections_from_file(&path, &store).unwrap(); + let auth = loaded[0].external_config.as_ref().and_then(|value| value.get("auth")).expect("restored MQ auth"); + assert_eq!(auth.get("token").and_then(serde_json::Value::as_str), Some("mq-token-secret")); + } + + #[test] + fn save_connections_moves_mq_basic_and_oauth_secrets_to_secret_store_and_restores_them() { + let path = temp_connections_file("mq-auth-multiple"); + let store = MemorySecretStore::default(); + let mut basic = connection("basic", "", ""); + basic.db_type = DatabaseType::MessageQueue; + basic.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://localhost:8080", + "auth": { + "kind": "basic", + "username": "admin", + "password": "basic-secret" + } + })); + let mut oauth = connection("oauth", "", ""); + oauth.db_type = DatabaseType::MessageQueue; + oauth.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://localhost:8080", + "auth": { + "kind": "oauth2", + "issuerUrl": "https://issuer/token", + "clientId": "client", + "clientSecret": "oauth-secret" + } + })); + + save_connections_to_file(&path, &[basic, oauth], &store).unwrap(); + + assert_eq!(store.get_existing("basic", "mq.auth.password").as_deref(), Some("basic-secret")); + assert_eq!(store.get_existing("oauth", "mq.auth.client_secret").as_deref(), Some("oauth-secret")); + let persisted_json = std::fs::read_to_string(&path).unwrap(); + assert!(!persisted_json.contains("basic-secret")); + assert!(!persisted_json.contains("oauth-secret")); + + let loaded = load_connections_from_file(&path, &store).unwrap(); + let basic_auth = loaded[0].external_config.as_ref().and_then(|value| value.get("auth")).unwrap(); + let oauth_auth = loaded[1].external_config.as_ref().and_then(|value| value.get("auth")).unwrap(); + assert_eq!(basic_auth.get("password").and_then(serde_json::Value::as_str), Some("basic-secret")); + assert_eq!(oauth_auth.get("clientSecret").and_then(serde_json::Value::as_str), Some("oauth-secret")); + } + + #[test] + fn save_connections_preserves_existing_mq_secret_when_config_is_sanitized() { + let path = temp_connections_file("mq-auth-preserve"); + let store = MemorySecretStore::default(); + store.set_existing("pulsar", "mq.auth.token", "existing-token"); + let mut config = connection("pulsar", "", ""); + config.db_type = DatabaseType::MessageQueue; + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://localhost:8080", + "auth": { + "kind": "token", + "token": "" + } + })); + + save_connections_to_file(&path, &[config], &store).unwrap(); + + assert_eq!(store.get_existing("pulsar", "mq.auth.token").as_deref(), Some("existing-token")); + let loaded = load_connections_from_file(&path, &store).unwrap(); + let auth = loaded[0].external_config.as_ref().and_then(|value| value.get("auth")).unwrap(); + assert_eq!(auth.get("token").and_then(serde_json::Value::as_str), Some("existing-token")); + } + + #[test] + fn save_connections_deletes_stale_mq_auth_secrets_when_kind_changes() { + let path = temp_connections_file("mq-auth-kind-change"); + let store = MemorySecretStore::default(); + store.set_existing("pulsar", MQ_AUTH_TOKEN_KEY, "old-token"); + let mut config = connection("pulsar", "", ""); + config.db_type = DatabaseType::MessageQueue; + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://localhost:8080", + "auth": { + "kind": "basic", + "username": "admin", + "password": "basic-secret" + } + })); + + save_connections_to_file(&path, &[config], &store).unwrap(); + + assert_eq!(store.get_existing("pulsar", MQ_AUTH_TOKEN_KEY), None); + assert_eq!(store.get_existing("pulsar", MQ_AUTH_PASSWORD_KEY).as_deref(), Some("basic-secret")); + } + + #[test] + fn save_connections_moves_mq_token_signing_key_to_secret_store_and_restores_it() { + let path = temp_connections_file("mq-token-signing"); + let store = MemorySecretStore::default(); + let mut config = connection("pulsar", "", ""); + config.db_type = DatabaseType::MessageQueue; + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://localhost:8080", + "auth": { "kind": "none" }, + "tokenSigning": { + "algorithm": "hs256", + "key": "broker-signing-secret" + } + })); + + save_connections_to_file(&path, &[config], &store).unwrap(); + + assert_eq!(store.get_existing("pulsar", MQ_TOKEN_SIGNING_KEY).as_deref(), Some("broker-signing-secret")); + let persisted_json = std::fs::read_to_string(&path).unwrap(); + assert!(!persisted_json.contains("broker-signing-secret")); + + let loaded = load_connections_from_file(&path, &store).unwrap(); + let signing = loaded[0].external_config.as_ref().and_then(|value| value.get("tokenSigning")).unwrap(); + assert_eq!(signing.get("key").and_then(serde_json::Value::as_str), Some("broker-signing-secret")); + } } diff --git a/crates/dbx-core/src/lib.rs b/crates/dbx-core/src/lib.rs index 76714d65b..a2e2da44a 100644 --- a/crates/dbx-core/src/lib.rs +++ b/crates/dbx-core/src/lib.rs @@ -25,6 +25,8 @@ pub mod history; pub mod jdbc; pub mod models; pub mod mongo_ops; +#[cfg(feature = "mq-admin")] +pub mod mq; pub mod object_source_sql; pub mod path_utils; pub mod plugins; diff --git a/crates/dbx-core/src/models/connection.rs b/crates/dbx-core/src/models/connection.rs index ae194f45b..d7bcfd0d8 100644 --- a/crates/dbx-core/src/models/connection.rs +++ b/crates/dbx-core/src/models/connection.rs @@ -2,7 +2,7 @@ use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC use serde::{Deserialize, Serialize}; use serde_json::Value; -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, PartialEq)] pub struct ConnectionConfig { pub id: String, pub name: String, @@ -301,6 +301,10 @@ pub enum DatabaseType { #[serde(rename = "questdb")] Questdb, Jdbc, + /// Message queue admin connection (Pulsar / Kafka / RocketMQ). The specific + /// system is determined by `external_config.systemKind`. + #[serde(rename = "mq")] + MessageQueue, } #[derive(Deserialize)] @@ -768,6 +772,7 @@ impl ConnectionConfig { format!("{scheme}://{host}:{port}") } DatabaseType::Jdbc => "jdbc:".to_string(), + DatabaseType::MessageQueue => self.message_queue_admin_url(), } } @@ -968,9 +973,21 @@ impl ConnectionConfig { DatabaseType::Jdbc => { self.connection_string.as_deref().filter(|value| !value.is_empty()).unwrap_or("jdbc:").to_string() } + DatabaseType::MessageQueue => self.message_queue_admin_url(), } } + fn message_queue_admin_url(&self) -> String { + self.external_config + .as_ref() + .and_then(|value| value.get("adminUrl").or_else(|| value.get("admin_url"))) + .and_then(|value| value.as_str()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or("mq://") + .to_string() + } + fn normalized_url_params(&self) -> String { let value = self.url_params.as_deref().unwrap_or("").trim(); if self.needs_bare_mysql() { diff --git a/crates/dbx-core/src/mq/README.md b/crates/dbx-core/src/mq/README.md new file mode 100644 index 000000000..b596f3552 --- /dev/null +++ b/crates/dbx-core/src/mq/README.md @@ -0,0 +1,54 @@ +# Message Queue Admin Module + +Apache Pulsar 管理 API 的 Rust 实现。 + +## 模块结构 + +``` +mq/ +├── mod.rs - MqAdminRegistry 注册表 +├── types.rs - 类型定义 +├── auth.rs - 认证机制 (OAuth2 缓存) +├── config.rs - 配置解析 +├── port.rs - MessageQueueAdmin trait +├── service.rs - 服务层函数 +└── adapters/ + ├── pulsar.rs - Pulsar 实现 + └── pulsar_version.rs - 版本探测 +``` + +## 核心 Trait + +```rust +#[async_trait] +pub trait MessageQueueAdmin: Send + Sync { + async fn list_tenants(&self) -> Result, String>; + async fn list_namespaces(&self, tenant: &str) -> Result, String>; + async fn list_topics(&self, ns: &NamespaceRef, opts: ListTopicsOpts) -> Result, String>; + // ... 45 more methods +} +``` + +## 使用示例 + +```rust +// AppState 会在存在 SSH/Proxy transport layer 时,把 Admin URL 改写到本地转发端点。 +let config = state.mq_admin_config_for_connection(&connection.id, &connection).await?; + +let adapter = state.mq_registry.get_or_build_config(&connection.id, config).await?; +let tenants = adapter.list_tenants().await?; +``` + +通常业务代码优先复用 `service.rs` 中的 `mq_*_core` 函数;只有需要直接组合底层能力时才访问 `MqAdminRegistry`。 + +## Feature Flag + +```toml +[features] +default = ["mq-admin"] +mq-admin = [] +``` + +## 文档 + +完整文档请参考:`docs/mq-index.md` diff --git a/crates/dbx-core/src/mq/adapters/mod.rs b/crates/dbx-core/src/mq/adapters/mod.rs new file mode 100644 index 000000000..4d3f768cc --- /dev/null +++ b/crates/dbx-core/src/mq/adapters/mod.rs @@ -0,0 +1,4 @@ +//! Concrete message queue admin adapters. + +pub mod pulsar; +pub mod pulsar_version; diff --git a/crates/dbx-core/src/mq/adapters/pulsar.rs b/crates/dbx-core/src/mq/adapters/pulsar.rs new file mode 100644 index 000000000..f9a635f14 --- /dev/null +++ b/crates/dbx-core/src/mq/adapters/pulsar.rs @@ -0,0 +1,1626 @@ +//! Apache Pulsar admin adapter. Talks to the Pulsar Admin REST API directly via +//! `reqwest` (no heavyweight Pulsar client dependency). All version-specific +//! endpoint construction and response parsing is delegated to +//! [`PulsarApiProfile`], so the adapter body stays version-agnostic. + +use async_trait::async_trait; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use futures::{stream, StreamExt}; +use std::collections::HashMap; +use std::net::{IpAddr, SocketAddr}; + +use super::pulsar_version::{DetectedVersion, PulsarApiProfile, PulsarVersion, VersionDetection}; +use crate::mq::auth::{MqAuth, TokenCache}; +use crate::mq::config::{MqAdminConfig, MqConnectOverride}; +use crate::mq::port::MessageQueueAdmin; +use crate::mq::types::*; +use crate::mq::util::truncate; + +/// How long to wait for a single admin REST call. +const REQUEST_TIMEOUT_SECS: u64 = 30; +/// How long to wait for the initial version-probe during construction. +const PROBE_TIMEOUT_SECS: u64 = 8; +const DETAIL_REQUEST_CONCURRENCY: usize = 8; +const PARTITION_METADATA_CONCURRENCY: usize = 8; +const PEEK_REQUEST_CONCURRENCY: usize = 4; +/// Maximum response body size for raw requests (10 MB). +const MAX_RAW_RESPONSE_BYTES: usize = 10 * 1024 * 1024; + +pub struct PulsarAdmin { + base: String, + http: reqwest::Client, + auth: MqAuth, + token_cache: TokenCache, + profile: PulsarApiProfile, +} + +impl PulsarAdmin { + /// Build an adapter, probing the broker version (unless pinned) to resolve + /// the API profile. Probe failure degrades gracefully to the 3.1.x baseline. + pub async fn new(cfg: MqAdminConfig) -> Result { + let base = normalize_base(&cfg.admin_url); + let http = build_http_client(cfg.tls_skip_verify, cfg.connect_override.as_ref(), &cfg.admin_url)?; + let auth = cfg.auth.clone(); + let token_cache = TokenCache::default(); + + let detected = detect_version(&http, &base, &auth, &token_cache, cfg.pinned_version.as_deref()).await; + let profile = PulsarApiProfile::resolve(detected); + + log::info!( + "Pulsar admin connected to {} (version profile {}, detection {})", + redact_base(&base), + profile.label(), + profile.detection().as_str() + ); + + Ok(Self { base, http, auth, token_cache, profile }) + } + + fn url(&self, path: &str) -> String { + format!("{}{}", self.base, path) + } + + /// Issue a request, applying auth, and return the raw response. + async fn send( + &self, + method: reqwest::Method, + path: &str, + body: Option<&serde_json::Value>, + query: Option<&[(String, String)]>, + ) -> Result { + let mut req = self.http.request(method.clone(), self.url(path)); + if let Some(q) = query { + req = req.query(q); + } + if let Some(body) = body { + req = req.json(body); + } + req = self.auth.apply(req, &self.http, &self.token_cache).await?; + req.send().await.map_err(|e| format!("Pulsar admin request to {path} failed: {e}")) + } + + /// GET a path and deserialize the JSON body into `T`. + async fn get_json(&self, path: &str) -> Result { + let resp = self.send(reqwest::Method::GET, path, None, None).await?; + let resp = error_for_status(resp, path).await?; + resp.json::().await.map_err(|e| format!("Failed to parse response from {path}: {e}")) + } + + /// GET a path and return the parsed JSON value. + async fn get_value(&self, path: &str) -> Result { + self.get_json::(path).await + } + + /// GET an optional policy endpoint. Pulsar returns 404/405/204 or an empty + /// body for some unset or unsupported scoped policies; those should not make + /// the whole policy panel fail. + async fn get_optional_value(&self, path: &str) -> Result, String> { + let resp = self.send(reqwest::Method::GET, path, None, None).await?; + let status = resp.status(); + if matches!( + status, + reqwest::StatusCode::NOT_FOUND | reqwest::StatusCode::METHOD_NOT_ALLOWED | reqwest::StatusCode::NO_CONTENT + ) { + return Ok(None); + } + let resp = error_for_status(resp, path).await?; + let body = resp.text().await.map_err(|e| format!("Failed to read response from {path}: {e}"))?; + let body = body.trim(); + if body.is_empty() { + return Ok(None); + } + serde_json::from_str(body).map(Some).map_err(|e| format!("Failed to parse response from {path}: {e}")) + } + + /// Send a mutating request that returns no meaningful body. + async fn send_empty( + &self, + method: reqwest::Method, + path: &str, + body: Option<&serde_json::Value>, + query: Option<&[(String, String)]>, + ) -> Result<(), String> { + let resp = self.send(method, path, body, query).await?; + error_for_status(resp, path).await?; + Ok(()) + } + + async fn namespace_admin_roles(&self, tenant: &str, namespace: &str) -> Result, String> { + let path = self.profile.namespace_permissions_path(tenant, namespace); + let raw: serde_json::Value = self.get_value(&path).await?; + let mut roles = raw.as_object().map(|obj| obj.keys().cloned().collect::>()).unwrap_or_default(); + roles.sort(); + Ok(roles) + } +} + +#[async_trait] +impl MessageQueueAdmin for PulsarAdmin { + fn capabilities(&self) -> MqCapabilities { + self.profile.capabilities() + } + + fn system_kind(&self) -> MqSystemKind { + MqSystemKind::Pulsar + } + + async fn test_connection(&self) -> Result { + // Verify reachability and authorization by listing clusters. + let clusters = self.get_value(&self.profile.clusters_path()).await?; + Ok(MqClusterInfo { + system_kind: MqSystemKind::Pulsar, + server_version: self.profile.raw_version().map(str::to_string), + resolved_profile: self.profile.label().to_string(), + version_detection: self.profile.detection().as_str().to_string(), + capabilities: self.profile.capabilities(), + extra: serde_json::json!({ "clusters": clusters }), + }) + } + + // ---- Tenants ---- + + async fn list_tenants(&self) -> Result, String> { + let names: Vec = self.get_json(&self.profile.tenants_path()).await?; + stream::iter(names.into_iter()) + .map(|name| async move { + match self.get_tenant(&name).await { + Ok(tenant) => Ok(tenant), + Err(err) if is_unsupported_endpoint_error(&err) => { + log::debug!("Pulsar tenant detail unavailable for '{}': {}", name, err); + Ok(TenantInfo { name, ..Default::default() }) + } + Err(err) => Err(err), + } + }) + .buffered(DETAIL_REQUEST_CONCURRENCY) + .collect::>() + .await + .into_iter() + .collect() + } + + async fn get_tenant(&self, name: &str) -> Result { + let raw = self.get_value(&self.profile.tenant_path(name)).await?; + Ok(TenantInfo { + name: name.to_string(), + admin_roles: string_array(&raw, "adminRoles"), + allowed_clusters: string_array(&raw, "allowedClusters"), + }) + } + + async fn create_tenant(&self, name: &str, cfg: TenantConfig) -> Result<(), String> { + let body = serde_json::json!({ + "adminRoles": cfg.admin_roles, + "allowedClusters": cfg.allowed_clusters, + }); + self.send_empty(reqwest::Method::PUT, &self.profile.tenant_path(name), Some(&body), None).await + } + + async fn update_tenant(&self, name: &str, cfg: TenantConfig) -> Result<(), String> { + let body = serde_json::json!({ + "adminRoles": cfg.admin_roles, + "allowedClusters": cfg.allowed_clusters, + }); + self.send_empty(reqwest::Method::POST, &self.profile.tenant_path(name), Some(&body), None).await + } + + async fn delete_tenant(&self, name: &str, force: bool) -> Result<(), String> { + let query = force_query(force); + self.send_empty(reqwest::Method::DELETE, &self.profile.tenant_path(name), None, query.as_deref()).await + } + + // ---- Namespaces ---- + + async fn list_namespaces(&self, tenant: &str) -> Result, String> { + // Returns fully-qualified `tenant/namespace` strings. + let names: Vec = self.get_json(&self.profile.namespaces_path(tenant)).await?; + stream::iter(names.into_iter()) + .map(|full| async move { + let namespace = full.rsplit('/').next().unwrap_or(&full).to_string(); + let admin_roles = match self.namespace_admin_roles(tenant, &namespace).await { + Ok(roles) => roles, + Err(err) if is_unsupported_endpoint_error(&err) => { + log::debug!("Pulsar namespace roles unavailable for '{}': {}", full, err); + Vec::new() + } + Err(err) => return Err(err), + }; + Ok(NamespaceInfo { tenant: tenant.to_string(), namespace, admin_roles }) + }) + .buffered(DETAIL_REQUEST_CONCURRENCY) + .collect::>() + .await + .into_iter() + .collect() + } + + async fn create_namespace(&self, ns: &NamespaceRef, cfg: NamespaceConfig) -> Result<(), String> { + let path = self.profile.namespace_path(&ns.tenant, &ns.namespace); + // An empty body creates a namespace with defaults; provide a policies + // object when clusters/bundles are specified. + let body = if cfg.clusters.is_empty() && cfg.bundles.is_none() { + None + } else { + let mut obj = serde_json::Map::new(); + if !cfg.clusters.is_empty() { + obj.insert("replication_clusters".to_string(), serde_json::json!(cfg.clusters)); + } + if let Some(bundles) = cfg.bundles { + obj.insert("bundles".to_string(), serde_json::json!({ "numBundles": bundles })); + } + Some(serde_json::Value::Object(obj)) + }; + self.send_empty(reqwest::Method::PUT, &path, body.as_ref(), None).await + } + + async fn delete_namespace(&self, ns: &NamespaceRef, force: bool) -> Result<(), String> { + let path = self.profile.namespace_path(&ns.tenant, &ns.namespace); + let query = force_query(force); + self.send_empty(reqwest::Method::DELETE, &path, None, query.as_deref()).await + } + + async fn get_namespace_policies(&self, ns: &NamespaceRef) -> Result { + self.get_value(&self.profile.namespace_policies_path(&ns.tenant, &ns.namespace)).await + } + + // ---- Topics ---- + + async fn list_topics(&self, ns: &NamespaceRef, opts: ListTopicsOpts) -> Result, String> { + let mut topics: Vec = Vec::new(); + let mut domains = vec![true]; + if opts.include_non_persistent { + domains.push(false); + } + + for persistent in domains { + // Partitioned topics first, so we can mark them and avoid double-listing. + let partitioned_path = self.profile.partitioned_topics_path(&ns.tenant, &ns.namespace, persistent); + let partitioned: Vec = match self.get_json(&partitioned_path).await { + Ok(partitioned) => partitioned, + Err(err) if is_unsupported_endpoint_error(&err) => Vec::new(), + Err(err) => return Err(err), + }; + let partitioned_set: std::collections::HashSet = partitioned.iter().cloned().collect(); + let domain = if persistent { "persistent" } else { "non-persistent" }; + let partitioned_topics = stream::iter(partitioned.into_iter()) + .map(|full| async move { + let partitions = self.partition_count_for_topic(domain, &full).await?; + Ok::<_, String>(TopicInfo { + short_name: short_topic_name(&full), + name: full, + partitioned: true, + partitions, + persistent, + }) + }) + .buffered(PARTITION_METADATA_CONCURRENCY) + .collect::>() + .await + .into_iter() + .collect::, _>>()?; + topics.extend(partitioned_topics); + + let plain: Vec = + self.get_json(&self.profile.topics_path(&ns.tenant, &ns.namespace, persistent)).await?; + for full in plain { + // Skip the per-partition entries of partitioned topics + // (`...-partition-0`) and the partitioned base names already added. + if partitioned_set.contains(&full) || is_partition_member(&full, &partitioned_set) { + continue; + } + topics.push(TopicInfo { + short_name: short_topic_name(&full), + name: full, + partitioned: false, + partitions: None, + persistent, + }); + } + } + + Ok(topics) + } + + async fn create_topic(&self, topic: &TopicRef, partitions: Option) -> Result<(), String> { + match partitions { + Some(count) if count >= 1 => { + let path = self.profile.topic_partitions_path(topic.domain(), &topic.path()); + let body = serde_json::json!(count); + self.send_empty(reqwest::Method::PUT, &path, Some(&body), None).await + } + _ => { + let path = self.profile.topic_base_path(topic.domain(), &topic.path()); + self.send_empty(reqwest::Method::PUT, &path, None, None).await + } + } + } + + async fn delete_topic(&self, topic: &TopicRef, force: bool) -> Result<(), String> { + // Try the partitioned-topic delete first; if the topic is not + // partitioned the broker returns 404, so fall back to a plain delete. + let partitions_path = self.profile.topic_partitions_path(topic.domain(), &topic.path()); + let query = force_query(force); + let resp = self.send(reqwest::Method::DELETE, &partitions_path, None, query.as_deref()).await?; + if resp.status().is_success() { + return Ok(()); + } + if resp.status().as_u16() != 404 { + return Err(status_error(resp, &partitions_path).await); + } + let base_path = self.profile.topic_base_path(topic.domain(), &topic.path()); + self.send_empty(reqwest::Method::DELETE, &base_path, None, query.as_deref()).await + } + + async fn update_partitions(&self, topic: &TopicRef, partitions: u32) -> Result<(), String> { + let path = self.profile.topic_partitions_path(topic.domain(), &topic.path()); + let body = serde_json::json!(partitions); + self.send_empty(reqwest::Method::POST, &path, Some(&body), None).await + } + + async fn get_topic_stats(&self, topic: &TopicRef) -> Result { + let raw = self.topic_stats_value(topic).await?; + Ok(self.profile.parse_topic_stats(&raw)) + } + + async fn get_topic_internal_stats(&self, topic: &TopicRef) -> Result { + let path = self.profile.topic_internal_stats_path(topic.domain(), &topic.path()); + self.get_value(&path).await + } + + // ---- Subscriptions ---- + + async fn list_subscriptions(&self, topic: &TopicRef) -> Result, String> { + // The stats payload carries richer per-subscription data than the bare + // subscriptions list endpoint, so prefer it and parse from there. + match self.topic_stats_value(topic).await { + Ok(raw) => { + let parsed = self.profile.parse_subscriptions(&raw); + if !parsed.is_empty() { + return Ok(parsed); + } + // Fall through to the names-only endpoint when stats has no subs + // (e.g. for partitioned topics, stats may be partition-scoped). + } + Err(err) => { + log::debug!("topic stats unavailable for subscription listing, falling back to names: {err}"); + } + } + + let names: Vec = self.get_json(&self.profile.subscriptions_path(topic.domain(), &topic.path())).await?; + Ok(names.into_iter().map(|name| SubscriptionInfo { name, ..Default::default() }).collect()) + } + + async fn create_subscription(&self, topic: &TopicRef, sub: &str, pos: ResetPosition) -> Result<(), String> { + // PUT .../subscription/{sub} with the desired message id in the body. + let path = self.profile.subscription_path(topic.domain(), &topic.path(), sub); + let body = reset_position_message_id(&pos); + self.send_empty(reqwest::Method::PUT, &path, Some(&body), None).await + } + + async fn delete_subscription(&self, topic: &TopicRef, sub: &str, force: bool) -> Result<(), String> { + let path = self.profile.subscription_path(topic.domain(), &topic.path(), sub); + let query = force_query(force); + self.send_empty(reqwest::Method::DELETE, &path, None, query.as_deref()).await + } + + async fn skip_messages(&self, topic: &TopicRef, sub: &str, count: SkipCount) -> Result<(), String> { + let path = match count { + SkipCount::All => self.profile.subscription_skip_all_path(topic.domain(), &topic.path(), sub), + SkipCount::Count { count } => { + self.profile.subscription_skip_path(topic.domain(), &topic.path(), sub, count) + } + }; + self.send_empty(reqwest::Method::POST, &path, None, None).await + } + + async fn reset_cursor(&self, topic: &TopicRef, sub: &str, pos: ResetPosition) -> Result<(), String> { + match pos { + ResetPosition::Timestamp { timestamp_ms } => { + // POST .../resetcursor/{timestamp} + let path = format!( + "{}/{}", + self.profile.subscription_reset_cursor_path(topic.domain(), &topic.path(), sub), + timestamp_ms + ); + self.send_empty(reqwest::Method::POST, &path, None, None).await + } + ResetPosition::MessageId { .. } | ResetPosition::Earliest | ResetPosition::Latest => { + // POST .../resetcursor with a message id body. + let path = self.profile.subscription_reset_cursor_path(topic.domain(), &topic.path(), sub); + let body = reset_position_message_id(&pos); + self.send_empty(reqwest::Method::POST, &path, Some(&body), None).await + } + } + } + + async fn clear_backlog(&self, topic: &TopicRef, sub: &str) -> Result<(), String> { + // Clearing backlog == skipping all messages on the subscription. + let path = self.profile.subscription_skip_all_path(topic.domain(), &topic.path(), sub); + self.send_empty(reqwest::Method::POST, &path, None, None).await + } + + async fn peek_messages(&self, topic: &TopicRef, sub: &str, count: u32) -> Result, String> { + if count == 0 { + return Ok(Vec::new()); + } + + let messages: Vec = stream::iter(1..=count) + .map(|position| async move { + let path = self.profile.subscription_peek_path(topic.domain(), &topic.path(), sub, position); + let resp = self.send(reqwest::Method::GET, &path, None, None).await?; + if resp.status().as_u16() == 404 && position > 1 { + return Ok::, String>(None); + } + let resp = error_for_status(resp, &path).await?; + let headers = response_headers(resp.headers()); + let payload = resp.bytes().await.map_err(|e| format!("Failed to read peeked message body: {e}"))?; + Ok(Some(peeked_message_from_response(position, headers, payload.as_ref()))) + }) + .buffered(PEEK_REQUEST_CONCURRENCY) + .collect::>() + .await + .into_iter() + .collect::, String>>()? + .into_iter() + .flatten() + .collect(); + + Ok(messages) + } + + async fn expire_messages(&self, topic: &TopicRef, sub: &str, expire_seconds: i64) -> Result<(), String> { + let path = self.profile.subscription_expire_path(topic.domain(), &topic.path(), sub, expire_seconds); + self.send_empty(reqwest::Method::POST, &path, None, None).await + } + + // ---- Producers / consumers ---- + + async fn list_producers(&self, topic: &TopicRef) -> Result, String> { + let raw = self.topic_stats_value(topic).await?; + Ok(producers_from_stats_payload(&raw)) + } + + async fn list_consumers(&self, topic: &TopicRef, sub: &str) -> Result, String> { + let subs = self.list_subscriptions(topic).await?; + Ok(subs.into_iter().find(|s| s.name == sub).map(|s| s.consumers).unwrap_or_default()) + } + + async fn unload_topic(&self, topic: &TopicRef) -> Result<(), String> { + let path = format!("{}/unload", self.profile.topic_base_path(topic.domain(), &topic.path())); + self.send_empty(reqwest::Method::PUT, &path, None, None).await + } + + // ---- Rate limits / quotas / retention ---- + + async fn set_publish_rate(&self, scope: &PolicyScope, rate: PublishRate) -> Result<(), String> { + let path = match scope { + PolicyScope::Namespace { tenant, namespace } => self.profile.namespace_publish_rate_path(tenant, namespace), + PolicyScope::Topic { tenant, namespace, topic, persistent } => { + let topic_ref = TopicRef { + tenant: tenant.clone(), + namespace: namespace.clone(), + topic: topic.clone(), + persistent: *persistent, + partitioned: None, + }; + self.profile.topic_publish_rate_path(topic_ref.domain(), &topic_ref.path()) + } + }; + let body = serde_json::to_value(rate).map_err(|e| e.to_string())?; + self.send_empty(reqwest::Method::POST, &path, Some(&body), None).await + } + + async fn set_dispatch_rate(&self, scope: &PolicyScope, rate: DispatchRate) -> Result<(), String> { + let path = match scope { + PolicyScope::Namespace { tenant, namespace } => { + self.profile.namespace_dispatch_rate_path(tenant, namespace) + } + PolicyScope::Topic { tenant, namespace, topic, persistent } => { + let topic_ref = TopicRef { + tenant: tenant.clone(), + namespace: namespace.clone(), + topic: topic.clone(), + persistent: *persistent, + partitioned: None, + }; + self.profile.topic_dispatch_rate_path(topic_ref.domain(), &topic_ref.path()) + } + }; + let body = serde_json::to_value(rate).map_err(|e| e.to_string())?; + self.send_empty(reqwest::Method::POST, &path, Some(&body), None).await + } + + async fn set_subscribe_rate(&self, scope: &PolicyScope, rate: SubscribeRate) -> Result<(), String> { + let path = match scope { + PolicyScope::Namespace { tenant, namespace } => { + self.profile.namespace_subscribe_rate_path(tenant, namespace) + } + PolicyScope::Topic { .. } => { + return Err("subscribe rate is only configurable at the namespace level".to_string()); + } + }; + let body = serde_json::to_value(rate).map_err(|e| e.to_string())?; + self.send_empty(reqwest::Method::POST, &path, Some(&body), None).await + } + + async fn set_backlog_quota(&self, scope: &PolicyScope, quota: BacklogQuota) -> Result<(), String> { + let path = match scope { + PolicyScope::Namespace { tenant, namespace } => { + self.profile.namespace_backlog_quota_path(tenant, namespace) + } + PolicyScope::Topic { tenant, namespace, topic, persistent } => { + let topic_ref = TopicRef { + tenant: tenant.clone(), + namespace: namespace.clone(), + topic: topic.clone(), + persistent: *persistent, + partitioned: None, + }; + self.profile.topic_backlog_quota_path(topic_ref.domain(), &topic_ref.path()) + } + }; + // `backlogQuotaType` goes in the query string; the limit object in the body. + let query = vec![("backlogQuotaType".to_string(), quota.quota_type.clone())]; + let body = serde_json::json!({ + "limit": quota.limit_size, + "limitSize": quota.limit_size, + "limitTime": quota.limit_time, + "policy": quota.policy, + }); + self.send_empty(reqwest::Method::POST, &path, Some(&body), Some(&query)).await + } + + async fn set_retention(&self, scope: &PolicyScope, retention: RetentionPolicy) -> Result<(), String> { + let path = match scope { + PolicyScope::Namespace { tenant, namespace } => self.profile.namespace_retention_path(tenant, namespace), + PolicyScope::Topic { tenant, namespace, topic, persistent } => { + let topic_ref = TopicRef { + tenant: tenant.clone(), + namespace: namespace.clone(), + topic: topic.clone(), + persistent: *persistent, + partitioned: None, + }; + self.profile.topic_retention_path(topic_ref.domain(), &topic_ref.path()) + } + }; + let body = serde_json::json!({ + "retentionTimeInMinutes": retention.retention_time_in_minutes, + "retentionSizeInMB": retention.retention_size_in_mb, + }); + self.send_empty(reqwest::Method::POST, &path, Some(&body), None).await + } + + async fn get_effective_policies(&self, scope: &PolicyScope) -> Result { + match scope { + PolicyScope::Namespace { tenant, namespace } => { + self.get_value(&self.profile.namespace_policies_path(tenant, namespace)).await + } + PolicyScope::Topic { tenant, namespace, topic, persistent } => { + let topic_ref = TopicRef { + tenant: tenant.clone(), + namespace: namespace.clone(), + topic: topic.clone(), + persistent: *persistent, + partitioned: None, + }; + let namespace_policies = self + .get_value(&self.profile.namespace_policies_path(&topic_ref.tenant, &topic_ref.namespace)) + .await?; + let topic_path = topic_ref.path(); + let publish_rate = self + .get_optional_value(&self.profile.topic_publish_rate_path(topic_ref.domain(), &topic_path)) + .await?; + let dispatch_rate = self + .get_optional_value(&self.profile.topic_dispatch_rate_path(topic_ref.domain(), &topic_path)) + .await?; + let backlog_quota = self + .get_optional_value(&self.profile.topic_backlog_quota_path(topic_ref.domain(), &topic_path)) + .await?; + let retention = self + .get_optional_value(&self.profile.topic_retention_path(topic_ref.domain(), &topic_path)) + .await?; + + Ok(serde_json::json!({ + "level": "topic", + "namespacePolicies": namespace_policies, + "topicPolicies": { + "publishRate": publish_rate, + "dispatchRate": dispatch_rate, + "backlogQuota": backlog_quota, + "retention": retention, + } + })) + } + } + } + + // ---- Permissions ---- + + async fn grant_permission(&self, scope: &PolicyScope, role: &str, actions: Vec) -> Result<(), String> { + let path = match scope { + PolicyScope::Namespace { tenant, namespace } => { + self.profile.namespace_permission_role_path(tenant, namespace, role) + } + PolicyScope::Topic { tenant, namespace, topic, persistent } => { + let topic_ref = TopicRef { + tenant: tenant.clone(), + namespace: namespace.clone(), + topic: topic.clone(), + persistent: *persistent, + partitioned: None, + }; + self.profile.topic_permission_role_path(topic_ref.domain(), &topic_ref.path(), role) + } + }; + let body = serde_json::to_value(&actions).map_err(|e| e.to_string())?; + self.send_empty(reqwest::Method::POST, &path, Some(&body), None).await + } + + async fn revoke_permission(&self, scope: &PolicyScope, role: &str) -> Result<(), String> { + let path = match scope { + PolicyScope::Namespace { tenant, namespace } => { + self.profile.namespace_permission_role_path(tenant, namespace, role) + } + PolicyScope::Topic { tenant, namespace, topic, persistent } => { + let topic_ref = TopicRef { + tenant: tenant.clone(), + namespace: namespace.clone(), + topic: topic.clone(), + persistent: *persistent, + partitioned: None, + }; + self.profile.topic_permission_role_path(topic_ref.domain(), &topic_ref.path(), role) + } + }; + self.send_empty(reqwest::Method::DELETE, &path, None, None).await + } + + async fn list_permissions(&self, scope: &PolicyScope) -> Result { + let path = match scope { + PolicyScope::Namespace { tenant, namespace } => self.profile.namespace_permissions_path(tenant, namespace), + PolicyScope::Topic { tenant, namespace, topic, persistent } => { + let topic_ref = TopicRef { + tenant: tenant.clone(), + namespace: namespace.clone(), + topic: topic.clone(), + persistent: *persistent, + partitioned: None, + }; + self.profile.topic_permissions_path(topic_ref.domain(), &topic_ref.path()) + } + }; + let raw: serde_json::Value = self.get_value(&path).await?; + let mut map = PermissionMap::new(); + if let Some(obj) = raw.as_object() { + for (role, actions) in obj { + let parsed: Vec = actions + .as_array() + .map(|arr| arr.iter().filter_map(|a| serde_json::from_value(a.clone()).ok()).collect()) + .unwrap_or_default(); + map.insert(role.clone(), parsed); + } + } + Ok(map) + } + + // ---- Monitoring ---- + + async fn get_backlog(&self, topic: &TopicRef, sub: Option<&str>) -> Result { + let stats = self.get_topic_stats(topic).await?; + Ok(backlog_stats_from_topic_stats(&self.profile, &stats, sub)) + } + + // ---- Escape hatch ---- + + async fn raw_request(&self, req: MqRawRequest) -> Result { + let method = reqwest::Method::from_bytes(req.method.to_ascii_uppercase().as_bytes()) + .map_err(|_| format!("Invalid HTTP method: {}", req.method))?; + // SSRF guard: only a relative path is allowed; it is appended to the + // connection's admin base. Reject absolute URLs and parent traversal. + let path = sanitize_raw_path(&req.path)?; + let query: Option> = req.query.map(|m| m.into_iter().collect::>()); + let resp = self.send(method, &path, req.body.as_ref(), query.as_deref()).await?; + let status = resp.status().as_u16(); + // Enforce a response size limit to prevent unbounded memory consumption. + let content_length = resp.content_length().unwrap_or(0) as usize; + if content_length > MAX_RAW_RESPONSE_BYTES { + return Err(format!( + "Raw response body exceeds the {MAX_RAW_RESPONSE_BYTES}-byte limit ({content_length} bytes reported)" + )); + } + let bytes = resp.bytes().await.map_err(|e| format!("Failed to read response body: {e}"))?; + if bytes.len() > MAX_RAW_RESPONSE_BYTES { + return Err(format!( + "Raw response body exceeds the {MAX_RAW_RESPONSE_BYTES}-byte limit ({} bytes received)", + bytes.len() + )); + } + let text = String::from_utf8_lossy(&bytes).to_string(); + match serde_json::from_str::(&text) { + Ok(body) => Ok(MqRawResponse { status, body, text: None }), + Err(_) => Ok(MqRawResponse { status, body: serde_json::Value::Null, text: Some(text) }), + } + } +} + +impl PulsarAdmin { + async fn topic_stats_value(&self, topic: &TopicRef) -> Result { + if topic.partitioned == Some(true) { + let partitioned_path = self.profile.topic_partitioned_stats_path(topic.domain(), &topic.path(), true); + return match self.get_value(&partitioned_path).await { + Ok(raw) => Ok(raw), + Err(partitioned_err) => { + let stats_path = self.profile.topic_stats_path(topic.domain(), &topic.path()); + self.get_value(&stats_path).await.map_err(|stats_err| { + format!( + "partitioned stats {partitioned_path} failed: {partitioned_err}; stats fallback {stats_path} failed: {stats_err}" + ) + }) + } + }; + } + + let stats_path = self.profile.topic_stats_path(topic.domain(), &topic.path()); + match self.get_value(&stats_path).await { + Ok(raw) => Ok(raw), + Err(err) if is_not_found_error(&err) => { + let partitioned_path = self.profile.topic_partitioned_stats_path(topic.domain(), &topic.path(), true); + self.get_value(&partitioned_path).await.map_err(|partitioned_err| { + format!("{err}; partitioned stats fallback {partitioned_path} failed: {partitioned_err}") + }) + } + Err(err) => Err(err), + } + } + + async fn partition_count_for_topic(&self, domain: &str, full_topic_name: &str) -> Result, String> { + let path = self.profile.topic_partitions_path(domain, topic_path_from_full_name(full_topic_name)); + match self.get_value(&path).await { + Ok(raw) => Ok(partition_count_from_metadata(&raw)), + Err(err) if is_unsupported_endpoint_error(&err) => { + log::debug!("partition metadata unavailable for {full_topic_name}: {err}"); + Ok(None) + } + Err(err) => Err(err), + } + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn build_http_client( + tls_skip_verify: bool, + connect_override: Option<&MqConnectOverride>, + admin_url: &str, +) -> Result { + let mut builder = crate::db::http_client_builder(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS)) + .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS)); + if tls_skip_verify { + builder = builder.danger_accept_invalid_certs(true); + } + if let Some(connect_override) = connect_override { + let url = reqwest::Url::parse(admin_url).map_err(|e| format!("MQ Admin URL is invalid: {e}"))?; + let original_host = url.host_str().ok_or("MQ Admin URL host is empty")?; + let override_ip = connect_override + .host + .parse::() + .map_err(|e| format!("MQ transport endpoint host must be an IP address: {e}"))?; + builder = builder.resolve(original_host, SocketAddr::new(override_ip, connect_override.port)); + } + builder.build().map_err(|e| format!("Failed to build HTTP client: {e}")) +} + +fn normalize_base(url: &str) -> String { + url.trim().trim_end_matches('/').to_string() +} + +/// Redact any userinfo in the base URL for logging. +fn redact_base(base: &str) -> String { + match base.split_once("://") { + Some((scheme, rest)) => match rest.split_once('@') { + Some((_, host)) => format!("{scheme}://***@{host}"), + None => base.to_string(), + }, + None => base.to_string(), + } +} + +/// Detect the broker version: honour a pinned version first, otherwise probe +/// `brokers/version`, otherwise fall back to the baseline profile. +async fn detect_version( + http: &reqwest::Client, + base: &str, + auth: &MqAuth, + cache: &TokenCache, + pinned: Option<&str>, +) -> DetectedVersion { + if let Some(pinned) = pinned.map(str::trim).filter(|s| !s.is_empty()) { + if let Some(version) = PulsarApiProfile::parse_version(pinned) { + return DetectedVersion { version, detection: VersionDetection::Pinned, raw: Some(pinned.to_string()) }; + } + log::warn!("Pinned Pulsar version '{pinned}' is not recognised; probing instead"); + } + + let url = format!("{base}/admin/v2/brokers/version"); + let req = http.get(&url).timeout(std::time::Duration::from_secs(PROBE_TIMEOUT_SECS)); + let probed = match auth.apply(req, http, cache).await { + Ok(req) => req.send().await, + Err(err) => { + log::warn!("Pulsar version probe auth failed: {err}; using baseline profile"); + return fallback_detection(); + } + }; + + match probed { + Ok(resp) if resp.status().is_success() => { + let raw = resp.text().await.unwrap_or_default(); + match PulsarApiProfile::parse_version(&raw) { + Some(version) => { + DetectedVersion { version, detection: VersionDetection::Probed, raw: Some(raw.trim().to_string()) } + } + None => { + log::info!("Pulsar broker reported version '{}', not first-party; using baseline", raw.trim()); + DetectedVersion { + version: PulsarVersion::V3_1, + detection: VersionDetection::Fallback, + raw: Some(raw.trim().to_string()), + } + } + } + } + Ok(resp) => { + log::warn!("Pulsar version probe returned {}; using baseline profile", resp.status()); + fallback_detection() + } + Err(err) => { + log::warn!("Pulsar version probe failed: {err}; using baseline profile"); + fallback_detection() + } + } +} + +fn fallback_detection() -> DetectedVersion { + DetectedVersion { version: PulsarVersion::V3_1, detection: VersionDetection::Fallback, raw: None } +} + +/// Return a `["force","true"]`-style query parameter list when `force` is set. +fn force_query(force: bool) -> Option> { + force.then(|| vec![("force".to_string(), "true".to_string())]) +} + +async fn error_for_status(resp: reqwest::Response, path: &str) -> Result { + if resp.status().is_success() { + Ok(resp) + } else { + Err(status_error(resp, path).await) + } +} + +async fn status_error(resp: reqwest::Response, path: &str) -> String { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + let detail = if body.trim().is_empty() { String::new() } else { format!(": {}", truncate(&body, 400)) }; + format!("Pulsar admin {path} returned {status}{detail}") +} + +fn string_array(value: &serde_json::Value, key: &str) -> Vec { + value + .get(key) + .and_then(serde_json::Value::as_array) + .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_string)).collect()) + .unwrap_or_default() +} + +/// The short topic name from a full name like +/// `persistent://public/default/orders` -> `orders`. +fn short_topic_name(full: &str) -> String { + let path = topic_path_from_full_name(full); + let mut parts = path.splitn(3, '/'); + let fallback = path.rsplit('/').next().unwrap_or(path); + match (parts.next(), parts.next(), parts.next()) { + (Some(_tenant), Some(_namespace), Some(topic)) => topic.to_string(), + _ => fallback.to_string(), + } +} + +fn topic_path_from_full_name(full: &str) -> &str { + full.split_once("://").map(|(_, path)| path).unwrap_or(full) +} + +fn partition_count_from_metadata(raw: &serde_json::Value) -> Option { + let count = raw.get("partitions").and_then(serde_json::Value::as_u64).or_else(|| raw.as_u64())?; + u32::try_from(count).ok().filter(|count| *count > 0) +} + +fn is_not_found_error(err: &str) -> bool { + err.contains(" returned 404 ") || err.contains("404 Not Found") +} + +fn is_method_not_allowed_error(err: &str) -> bool { + err.contains(" returned 405 ") || err.contains("405 Method Not Allowed") +} + +fn is_unsupported_endpoint_error(err: &str) -> bool { + is_not_found_error(err) || is_method_not_allowed_error(err) +} + +fn producers_from_stats_payload(raw: &serde_json::Value) -> Vec { + let mut producers = Vec::new(); + append_producers_from_stats(raw, &mut producers); + if let Some(partitions) = raw.get("partitions").and_then(serde_json::Value::as_object) { + for partition_stats in partitions.values() { + append_producers_from_stats(partition_stats, &mut producers); + } + } + producers +} + +fn append_producers_from_stats(raw: &serde_json::Value, producers: &mut Vec) { + if let Some(items) = raw.get("publishers").and_then(serde_json::Value::as_array) { + producers.extend(items.iter().map(parse_producer)); + } +} + +/// Whether `full` is a per-partition member (`...-partition-N`) of one of the +/// partitioned base topics. +fn is_partition_member(full: &str, partitioned: &std::collections::HashSet) -> bool { + if let Some(idx) = full.rfind("-partition-") { + let (base, suffix) = full.split_at(idx); + let n = &suffix["-partition-".len()..]; + if !n.is_empty() && n.chars().all(|c| c.is_ascii_digit()) { + return partitioned.contains(base); + } + } + false +} + +/// Build a message-id body for reset-cursor / create-subscription based on the +/// requested position. Earliest/latest use the sentinel ledger/entry ids Pulsar +/// recognises. +fn reset_position_message_id(pos: &ResetPosition) -> serde_json::Value { + match pos { + ResetPosition::Earliest => serde_json::json!({ "ledgerId": -1, "entryId": -1 }), + ResetPosition::Latest => serde_json::json!({ "ledgerId": i64::MAX, "entryId": i64::MAX }), + ResetPosition::MessageId { ledger_id, entry_id } => { + serde_json::json!({ "ledgerId": ledger_id, "entryId": entry_id }) + } + // Timestamp is handled by a dedicated path; default to latest here. + ResetPosition::Timestamp { .. } => serde_json::json!({ "ledgerId": i64::MAX, "entryId": i64::MAX }), + } +} + +fn parse_producer(v: &serde_json::Value) -> ProducerInfo { + ProducerInfo { + producer_id: v.get("producerId").and_then(serde_json::Value::as_i64).unwrap_or(0), + producer_name: v.get("producerName").and_then(serde_json::Value::as_str).unwrap_or("").to_string(), + msg_rate_in: v.get("msgRateIn").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + msg_throughput_in: v.get("msgThroughputIn").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + address: v.get("address").and_then(serde_json::Value::as_str).unwrap_or("").to_string(), + client_version: v.get("clientVersion").and_then(serde_json::Value::as_str).unwrap_or("").to_string(), + } +} + +fn response_headers(headers: &reqwest::header::HeaderMap) -> HashMap { + headers + .iter() + .filter_map(|(name, value)| { + value.to_str().ok().map(|value| (name.as_str().to_ascii_lowercase(), value.to_string())) + }) + .collect() +} + +fn peeked_message_from_response(position: u32, headers: HashMap, payload: &[u8]) -> PeekedMessage { + let properties = headers + .iter() + .filter_map(|(name, value)| { + name.strip_prefix("x-pulsar-property-").map(|property| (property.to_string(), value.clone())) + }) + .collect(); + PeekedMessage { + position, + message_id: header_value(&headers, &["x-pulsar-message-id", "x-pulsar-messageid"]), + key: header_value(&headers, &["x-pulsar-key", "x-pulsar-message-key"]), + publish_time: header_value(&headers, &["x-pulsar-publish-time", "x-pulsar-publish-time-ms"]), + event_time: header_value(&headers, &["x-pulsar-event-time", "x-pulsar-event-time-ms"]), + properties, + headers, + payload_base64: BASE64.encode(payload), + payload_text: std::str::from_utf8(payload).ok().map(str::to_string), + } +} + +fn header_value(headers: &HashMap, names: &[&str]) -> Option { + names.iter().find_map(|name| headers.get(*name).cloned()) +} + +/// Validate a raw request path: must be a relative path beginning with `/`, no +/// scheme/host, no `..` traversal. +fn sanitize_raw_path(path: &str) -> Result { + let trimmed = path.trim(); + if trimmed.is_empty() { + return Err("Raw request path is empty".to_string()); + } + if trimmed.contains("://") || trimmed.starts_with("//") { + return Err("Raw request path must be relative to the admin base (no scheme/host allowed)".to_string()); + } + if !trimmed.starts_with('/') { + return Err("Raw request path must start with '/'".to_string()); + } + if trimmed.split('/').any(|seg| seg == "..") { + return Err("Raw request path must not contain '..'".to_string()); + } + Ok(trimmed.to_string()) +} + +fn backlog_stats_from_topic_stats(profile: &PulsarApiProfile, stats: &TopicStats, sub: Option<&str>) -> BacklogStats { + let subs = profile.parse_subscriptions(&stats.raw); + let msg_backlog = match sub { + Some(name) => subs.into_iter().find(|s| s.name == name).map(|s| s.msg_backlog).unwrap_or(0), + None => subs.into_iter().fold(0_i64, |total, sub| total.saturating_add(sub.msg_backlog)), + }; + + BacklogStats { msg_backlog, backlog_size: stats.backlog_size } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::{TcpListener, TcpStream}; + use tokio::sync::Mutex; + + #[derive(Clone)] + struct MockRoute { + method: &'static str, + path: &'static str, + status: u16, + body: &'static str, + } + + #[derive(Clone, Debug)] + struct ObservedRequest { + method: String, + path: String, + body: String, + } + + struct MockPulsarServer { + base_url: String, + observed: std::sync::Arc>>, + } + + impl MockPulsarServer { + async fn spawn(routes: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("failed to bind test server socket"); + let addr = listener.local_addr().expect("failed to get test server local address"); + let routes = std::sync::Arc::new(routes); + let observed = std::sync::Arc::new(Mutex::new(Vec::new())); + let server_observed = observed.clone(); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + break; + }; + let routes = routes.clone(); + let observed = server_observed.clone(); + tokio::spawn(async move { + let Some(request) = read_mock_request(&mut stream).await else { + return; + }; + let response = routes + .iter() + .find(|route| route.method == request.method && route.path == request.path) + .cloned() + .unwrap_or(MockRoute { + method: "GET", + path: "", + status: 500, + body: r#"{"reason":"unexpected request"}"#, + }); + observed.lock().await.push(request); + write_mock_response(&mut stream, response.status, response.body).await; + }); + } + }); + + Self { base_url: format!("http://{addr}"), observed } + } + + async fn admin(&self) -> PulsarAdmin { + PulsarAdmin::new(MqAdminConfig { + system_kind: MqSystemKind::Pulsar, + admin_url: self.base_url.clone(), + auth: MqAuth::None, + tls_skip_verify: false, + pinned_version: Some("3.1".to_string()), + token_signing: None, + connect_override: None, + extra: serde_json::Value::Null, + }) + .await + .expect("failed to create test Pulsar admin client") + } + + async fn observed(&self) -> Vec { + self.observed.lock().await.clone() + } + } + + async fn read_mock_request(stream: &mut TcpStream) -> Option { + let mut buf = Vec::new(); + let mut tmp = [0_u8; 1024]; + let header_end = loop { + let n = stream.read(&mut tmp).await.ok()?; + if n == 0 { + return None; + } + buf.extend_from_slice(&tmp[..n]); + if let Some(index) = find_header_end(&buf) { + break index; + } + }; + + let header = String::from_utf8_lossy(&buf[..header_end]); + let mut lines = header.lines(); + let first = lines.next()?; + let mut parts = first.split_whitespace(); + let method = parts.next()?.to_string(); + let path = parts.next()?.to_string(); + let content_length = lines + .filter_map(|line| line.split_once(':')) + .find_map(|(name, value)| { + name.eq_ignore_ascii_case("content-length").then(|| value.trim().parse::().ok()).flatten() + }) + .unwrap_or(0); + + let body_start = header_end + 4; + let mut body = buf.get(body_start..).unwrap_or_default().to_vec(); + while body.len() < content_length { + let n = stream.read(&mut tmp).await.ok()?; + if n == 0 { + break; + } + body.extend_from_slice(&tmp[..n]); + } + body.truncate(content_length); + + Some(ObservedRequest { method, path, body: String::from_utf8_lossy(&body).to_string() }) + } + + fn find_header_end(buf: &[u8]) -> Option { + buf.windows(4).position(|window| window == b"\r\n\r\n") + } + + async fn write_mock_response(stream: &mut TcpStream, status: u16, body: &str) { + let reason = match status { + 200 => "OK", + 204 => "No Content", + 400 => "Bad Request", + 401 => "Unauthorized", + 403 => "Forbidden", + 404 => "Not Found", + 405 => "Method Not Allowed", + 500 => "Internal Server Error", + _ => "OK", + }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()).await; + } + + fn assert_json_body(request: &ObservedRequest, expected: serde_json::Value) { + let actual: serde_json::Value = + serde_json::from_str(&request.body).expect("test request body should be valid JSON"); + assert_eq!(actual, expected); + } + + #[test] + fn normalizes_base_url() { + assert_eq!(normalize_base("http://x:8080/"), "http://x:8080"); + assert_eq!(normalize_base(" http://x:8080 "), "http://x:8080"); + } + + #[test] + fn redacts_userinfo() { + assert_eq!(redact_base("http://user:pw@host:8080"), "http://***@host:8080"); + assert_eq!(redact_base("http://host:8080"), "http://host:8080"); + } + + #[test] + fn short_topic_name_extracts_last_segment() { + assert_eq!(short_topic_name("persistent://public/default/orders"), "orders"); + assert_eq!(short_topic_name("persistent://public/default/orders/a"), "orders/a"); + assert_eq!(short_topic_name("non-persistent://tenant/ns/topic/with/slash"), "topic/with/slash"); + } + + #[test] + fn partition_count_from_metadata_reads_pulsar_shapes() { + assert_eq!(partition_count_from_metadata(&serde_json::json!({ "partitions": 4 })), Some(4)); + assert_eq!(partition_count_from_metadata(&serde_json::json!(8)), Some(8)); + assert_eq!(partition_count_from_metadata(&serde_json::json!({ "partitions": 0 })), None); + } + + #[tokio::test] + async fn list_tenants_returns_detail_server_errors() { + let server = MockPulsarServer::spawn(vec![ + MockRoute { method: "GET", path: "/admin/v2/tenants", status: 200, body: r#"["acme"]"# }, + MockRoute { method: "GET", path: "/admin/v2/tenants/acme", status: 500, body: r#"{"reason":"boom"}"# }, + ]) + .await; + let admin = server.admin().await; + + let err = admin.list_tenants().await.unwrap_err(); + + assert!(err.contains("/admin/v2/tenants/acme")); + assert!(err.contains("500 Internal Server Error")); + } + + #[tokio::test] + async fn list_namespaces_returns_permission_server_errors() { + let server = MockPulsarServer::spawn(vec![ + MockRoute { method: "GET", path: "/admin/v2/namespaces/acme", status: 200, body: r#"["acme/events"]"# }, + MockRoute { + method: "GET", + path: "/admin/v2/namespaces/acme/events/permissions", + status: 403, + body: r#"{"reason":"denied"}"#, + }, + ]) + .await; + let admin = server.admin().await; + + let err = admin.list_namespaces("acme").await.unwrap_err(); + + assert!(err.contains("/admin/v2/namespaces/acme/events/permissions")); + assert!(err.contains("403 Forbidden")); + } + + #[tokio::test] + async fn list_topics_returns_partitioned_topic_list_server_errors() { + let server = MockPulsarServer::spawn(vec![ + MockRoute { + method: "GET", + path: "/admin/v2/persistent/public/default/partitioned", + status: 500, + body: r#"{"reason":"boom"}"#, + }, + MockRoute { method: "GET", path: "/admin/v2/persistent/public/default", status: 200, body: "[]" }, + ]) + .await; + let admin = server.admin().await; + + let err = admin + .list_topics( + &NamespaceRef { tenant: "public".to_string(), namespace: "default".to_string() }, + ListTopicsOpts { include_non_persistent: false }, + ) + .await + .unwrap_err(); + + assert!(err.contains("/admin/v2/persistent/public/default/partitioned")); + assert!(err.contains("500 Internal Server Error")); + } + + #[tokio::test] + async fn list_topics_allows_unsupported_partitioned_topic_endpoint() { + let server = MockPulsarServer::spawn(vec![ + MockRoute { + method: "GET", + path: "/admin/v2/persistent/public/default/partitioned", + status: 404, + body: r#"{"reason":"not enabled"}"#, + }, + MockRoute { + method: "GET", + path: "/admin/v2/persistent/public/default", + status: 200, + body: r#"["persistent://public/default/plain"]"#, + }, + ]) + .await; + let admin = server.admin().await; + + let topics = admin + .list_topics( + &NamespaceRef { tenant: "public".to_string(), namespace: "default".to_string() }, + ListTopicsOpts { include_non_persistent: false }, + ) + .await + .expect("list_topics should succeed"); + + assert_eq!(topics.len(), 1); + assert_eq!(topics[0].name, "persistent://public/default/plain"); + assert!(!topics[0].partitioned); + } + + #[tokio::test] + async fn list_topics_returns_partition_metadata_errors() { + let server = MockPulsarServer::spawn(vec![ + MockRoute { + method: "GET", + path: "/admin/v2/persistent/public/default/partitioned", + status: 200, + body: r#"["persistent://public/default/orders"]"#, + }, + MockRoute { + method: "GET", + path: "/admin/v2/persistent/public/default/orders/partitions", + status: 403, + body: r#"{"reason":"denied"}"#, + }, + MockRoute { method: "GET", path: "/admin/v2/persistent/public/default", status: 200, body: "[]" }, + ]) + .await; + let admin = server.admin().await; + + let err = admin + .list_topics( + &NamespaceRef { tenant: "public".to_string(), namespace: "default".to_string() }, + ListTopicsOpts { include_non_persistent: false }, + ) + .await + .unwrap_err(); + + assert!(err.contains("/admin/v2/persistent/public/default/orders/partitions")); + assert!(err.contains("403 Forbidden")); + } + + #[tokio::test] + async fn mutating_methods_use_expected_http_contracts() { + let server = MockPulsarServer::spawn(vec![ + MockRoute { method: "PUT", path: "/admin/v2/tenants/acme", status: 204, body: "" }, + MockRoute { method: "PUT", path: "/admin/v2/namespaces/acme/events", status: 204, body: "" }, + MockRoute { + method: "PUT", + path: "/admin/v2/persistent/acme/events/orders/partitions", + status: 204, + body: "", + }, + MockRoute { + method: "PUT", + path: "/admin/v2/persistent/acme/events/orders/subscription/sub%2Frole%20%231", + status: 204, + body: "", + }, + MockRoute { method: "POST", path: "/admin/v2/namespaces/acme/events/publishRate", status: 204, body: "" }, + MockRoute { method: "GET", path: "/admin/v2/clusters", status: 200, body: r#"["ec-pulsar"]"# }, + ]) + .await; + let admin = server.admin().await; + + admin + .create_tenant( + "acme", + TenantConfig { + admin_roles: vec!["admin".to_string()], + allowed_clusters: vec!["ec-pulsar".to_string()], + }, + ) + .await + .expect("create_tenant should succeed"); + admin + .create_namespace( + &NamespaceRef { tenant: "acme".to_string(), namespace: "events".to_string() }, + NamespaceConfig { clusters: vec!["ec-pulsar".to_string()], bundles: Some(16) }, + ) + .await + .expect("create_namespace should succeed"); + let topic = TopicRef { + tenant: "acme".to_string(), + namespace: "events".to_string(), + topic: "orders".to_string(), + persistent: true, + partitioned: Some(true), + }; + admin.create_topic(&topic, Some(3)).await.expect("create_topic should succeed"); + admin + .create_subscription(&topic, "sub/role #1", ResetPosition::Latest) + .await + .expect("create_subscription should succeed"); + admin + .set_publish_rate( + &PolicyScope::Namespace { tenant: "acme".to_string(), namespace: "events".to_string() }, + PublishRate { publish_throttling_rate_in_msg: 10, publish_throttling_rate_in_byte: 2048 }, + ) + .await + .expect("set_publish_rate should succeed"); + let raw = admin + .raw_request(MqRawRequest { + method: "GET".to_string(), + path: "/admin/v2/clusters".to_string(), + query: None, + body: None, + }) + .await + .expect("raw_admin_request should succeed"); + assert_eq!(raw.status, 200); + assert_eq!(raw.body, serde_json::json!(["ec-pulsar"])); + + let observed = server.observed().await; + let method_paths = observed.iter().map(|req| (req.method.as_str(), req.path.as_str())).collect::>(); + assert_eq!( + method_paths, + vec![ + ("PUT", "/admin/v2/tenants/acme"), + ("PUT", "/admin/v2/namespaces/acme/events"), + ("PUT", "/admin/v2/persistent/acme/events/orders/partitions"), + ("PUT", "/admin/v2/persistent/acme/events/orders/subscription/sub%2Frole%20%231"), + ("POST", "/admin/v2/namespaces/acme/events/publishRate"), + ("GET", "/admin/v2/clusters"), + ] + ); + assert_json_body( + &observed[0], + serde_json::json!({ "adminRoles": ["admin"], "allowedClusters": ["ec-pulsar"] }), + ); + assert_json_body( + &observed[1], + serde_json::json!({ "replication_clusters": ["ec-pulsar"], "bundles": { "numBundles": 16 } }), + ); + assert_json_body(&observed[2], serde_json::json!(3)); + assert_json_body(&observed[3], serde_json::json!({ "ledgerId": i64::MAX, "entryId": i64::MAX })); + assert_json_body( + &observed[4], + serde_json::json!({ "publishThrottlingRateInMsg": 10, "publishThrottlingRateInByte": 2048 }), + ); + assert!(observed[5].body.is_empty()); + } + + #[tokio::test] + async fn status_errors_include_path_status_and_body() { + let server = MockPulsarServer::spawn(vec![ + MockRoute { method: "GET", path: "/admin/v2/tenants", status: 401, body: r#"{"reason":"missing token"}"# }, + MockRoute { + method: "GET", + path: "/admin/v2/tenants/forbidden", + status: 403, + body: r#"{"reason":"denied"}"#, + }, + MockRoute { + method: "GET", + path: "/admin/v2/namespaces/public/missing", + status: 404, + body: r#"{"reason":"missing"}"#, + }, + MockRoute { method: "GET", path: "/admin/v2/clusters", status: 500, body: r#"{"reason":"broker down"}"# }, + ]) + .await; + let admin = server.admin().await; + + let unauthorized = admin.list_tenants().await.unwrap_err(); + assert!(unauthorized.contains("/admin/v2/tenants")); + assert!(unauthorized.contains("401 Unauthorized")); + assert!(unauthorized.contains("missing token")); + + let forbidden = admin.get_tenant("forbidden").await.unwrap_err(); + assert!(forbidden.contains("/admin/v2/tenants/forbidden")); + assert!(forbidden.contains("403 Forbidden")); + assert!(forbidden.contains("denied")); + + let missing = admin + .get_namespace_policies(&NamespaceRef { tenant: "public".to_string(), namespace: "missing".to_string() }) + .await + .unwrap_err(); + assert!(missing.contains("/admin/v2/namespaces/public/missing")); + assert!(missing.contains("404 Not Found")); + assert!(missing.contains("missing")); + + let server_error = admin.test_connection().await.unwrap_err(); + assert!(server_error.contains("/admin/v2/clusters")); + assert!(server_error.contains("500 Internal Server Error")); + assert!(server_error.contains("broker down")); + } + + #[test] + fn producers_from_stats_payload_reads_partitioned_stats() { + let raw = serde_json::json!({ + "publishers": [{ "producerId": 10, "producerName": "aggregate" }], + "partitions": { + "persistent://b2b/ec-product-service/goods_status_change-partition-0": { + "publishers": [{ "producerId": 1, "producerName": "p0" }] + }, + "persistent://b2b/ec-product-service/goods_status_change-partition-1": { + "publishers": [{ "producerId": 2, "producerName": "p1" }] + } + } + }); + + let producers = producers_from_stats_payload(&raw); + + assert_eq!( + producers.iter().map(|producer| producer.producer_name.as_str()).collect::>(), + vec!["aggregate", "p0", "p1"] + ); + } + + #[test] + fn detects_partition_members() { + let mut set = std::collections::HashSet::new(); + set.insert("persistent://public/default/orders".to_string()); + assert!(is_partition_member("persistent://public/default/orders-partition-0", &set)); + assert!(!is_partition_member("persistent://public/default/orders-partition-x", &set)); + assert!(!is_partition_member("persistent://public/default/other", &set)); + } + + #[test] + fn sanitizes_raw_paths() { + assert!(sanitize_raw_path("/admin/v2/tenants").is_ok()); + assert!(sanitize_raw_path("http://evil/x").is_err()); + assert!(sanitize_raw_path("//evil/x").is_err()); + assert!(sanitize_raw_path("admin/v2").is_err()); + assert!(sanitize_raw_path("/admin/../../etc").is_err()); + assert!(sanitize_raw_path(" ").is_err()); + } + + #[test] + fn topic_backlog_sums_subscription_backlogs() { + let profile = PulsarApiProfile::default_baseline(); + let stats = TopicStats { + msg_rate_in: 0.0, + msg_rate_out: 0.0, + msg_throughput_in: 0.0, + msg_throughput_out: 0.0, + storage_size: 0, + backlog_size: 4096, + msg_in_counter: 0, + msg_out_counter: 0, + subscription_count: 2, + producer_count: 0, + raw: serde_json::json!({ + "subscriptions": { + "sub-a": { "msgBacklog": 7 }, + "sub-b": { "msgBacklog": 11 } + } + }), + }; + + let backlog = backlog_stats_from_topic_stats(&profile, &stats, None); + + assert_eq!(backlog.msg_backlog, 18); + assert_eq!(backlog.backlog_size, 4096); + } + + #[test] + fn reset_position_message_ids() { + assert_eq!( + reset_position_message_id(&ResetPosition::Earliest), + serde_json::json!({"ledgerId": -1, "entryId": -1}) + ); + let mid = reset_position_message_id(&ResetPosition::MessageId { ledger_id: 5, entry_id: 9 }); + assert_eq!(mid, serde_json::json!({"ledgerId": 5, "entryId": 9})); + } + + #[test] + fn force_query_only_when_true() { + assert!(force_query(false).is_none()); + assert_eq!( + force_query(true).expect("force=true should produce query params"), + vec![("force".to_string(), "true".to_string())] + ); + } + + #[test] + fn builds_peeked_message_from_headers_and_payload() { + let message = peeked_message_from_response( + 2, + HashMap::from([ + ("x-pulsar-message-id".to_string(), "1:2:3".to_string()), + ("x-pulsar-key".to_string(), "order-1".to_string()), + ("x-pulsar-property-source".to_string(), "api".to_string()), + ]), + b"hello", + ); + + assert_eq!(message.position, 2); + assert_eq!(message.message_id.as_deref(), Some("1:2:3")); + assert_eq!(message.key.as_deref(), Some("order-1")); + assert_eq!(message.properties.get("source").map(String::as_str), Some("api")); + assert_eq!(message.payload_base64, "aGVsbG8="); + assert_eq!(message.payload_text.as_deref(), Some("hello")); + } +} diff --git a/crates/dbx-core/src/mq/adapters/pulsar_version.rs b/crates/dbx-core/src/mq/adapters/pulsar_version.rs new file mode 100644 index 000000000..563278f7c --- /dev/null +++ b/crates/dbx-core/src/mq/adapters/pulsar_version.rs @@ -0,0 +1,584 @@ +//! Pulsar version detection and the version "profile" that isolates all +//! version-specific behaviour (endpoint paths + response parsing + capability +//! flags) behind one object. Adapter methods express *what* to do; the profile +//! decides *how*, per the resolved version. +//! +//! First-party scope is **Pulsar 3.1.x** (REST-contract compatible with the +//! 3.0.x LTS line). Newer versions are handled additively: add a `PulsarVersion` +//! variant, add the differing path/parse branches here, and the adapter, service +//! layer, commands, routes, and frontend stay untouched. See +//! `docs/message-queue-admin-design.zh-CN.md` chapter 10. + +use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; +use serde::{Deserialize, Serialize}; + +use super::super::types::{MqCapabilities, SubscriptionInfo, TopicStats}; + +/// A recognised Pulsar major/minor line. Unknown versions fall back to the +/// nearest known profile in conservative mode. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum PulsarVersion { + /// Pulsar 3.0.x / 3.1.x — the first-party baseline. + V3_1, +} + +impl PulsarVersion { + pub fn label(self) -> &'static str { + match self { + PulsarVersion::V3_1 => "3.1.x", + } + } +} + +/// How the version was determined. Surfaced to the UI so users understand +/// whether the adapter is running on a confirmed version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum VersionDetection { + /// Probed from `brokers/version`. + Probed, + /// Pinned by the user in connection config. + Pinned, + /// Could not determine; using the default baseline profile. + Fallback, +} + +impl VersionDetection { + pub fn as_str(self) -> &'static str { + match self { + VersionDetection::Probed => "probed", + VersionDetection::Pinned => "pinned", + VersionDetection::Fallback => "fallback", + } + } +} + +/// Resolved version picture: which version, how we found it, and the raw string. +#[derive(Debug, Clone)] +pub struct DetectedVersion { + pub version: PulsarVersion, + pub detection: VersionDetection, + pub raw: Option, +} + +/// The version profile: a single object that answers every version-dependent +/// question. Adapter methods route all path construction and response parsing +/// through here. +#[derive(Debug, Clone)] +pub struct PulsarApiProfile { + version: PulsarVersion, + detection: VersionDetection, + raw_version: Option, +} + +impl PulsarApiProfile { + /// Resolve a profile from a detected version. + pub fn resolve(detected: DetectedVersion) -> Self { + Self { version: detected.version, detection: detected.detection, raw_version: detected.raw } + } + + /// The default baseline profile (3.1.x), used when detection fails. + pub fn default_baseline() -> Self { + Self { version: PulsarVersion::V3_1, detection: VersionDetection::Fallback, raw_version: None } + } + + pub fn version(&self) -> PulsarVersion { + self.version + } + + pub fn detection(&self) -> VersionDetection { + self.detection + } + + pub fn raw_version(&self) -> Option<&str> { + self.raw_version.as_deref() + } + + pub fn label(&self) -> &'static str { + self.version.label() + } + + /// Parse a raw broker version string (e.g. `3.1.2`, `3.0.7`, or + /// `Pulsar version 3.1.2 ...`) into a known [`PulsarVersion`]. + pub fn parse_version(raw: &str) -> Option { + let trimmed = raw.trim(); + // Find the first dotted numeric token. + let token = trimmed.split_whitespace().find(|part| part.chars().next().is_some_and(|c| c.is_ascii_digit())); + let token = token.unwrap_or(trimmed); + let mut nums = token.split('.'); + let major: u32 = nums.next()?.trim().parse().ok()?; + let _minor: u32 = nums.next().and_then(|m| m.trim().parse().ok()).unwrap_or(0); + // 3.x (and the 3.0/3.1 LTS family) map onto the V3_1 baseline. + match major { + 3 => Some(PulsarVersion::V3_1), + // Newer majors are not yet first-party; caller decides fallback. + _ => None, + } + } + + /// Capability flags for this version. Older versions missing endpoints would + /// turn the corresponding flag off here so the UI hides them. + pub fn capabilities(&self) -> MqCapabilities { + match self.version { + PulsarVersion::V3_1 => MqCapabilities { + supports_tenants: true, + supports_namespaces: true, + supports_partitioned_topics: true, + supports_subscriptions: true, + supports_create_subscription: true, + supports_reset_cursor: true, + supports_skip_messages: true, + supports_clear_backlog: true, + supports_peek_messages: true, + supports_expire_messages: true, + supports_rate_limits: true, + supports_backlog_quota: true, + supports_retention: true, + supports_permissions: true, + supports_geo_replication: true, + // Token signing is NOT a broker REST capability — see chapter 11. + supports_token_management: false, + supports_raw_admin_api: true, + }, + } + } + + // ---- Path construction (all relative to the admin base) ---- + + pub fn broker_version_path(&self) -> String { + "/admin/v2/brokers/version".to_string() + } + + pub fn clusters_path(&self) -> String { + "/admin/v2/clusters".to_string() + } + + pub fn tenants_path(&self) -> String { + "/admin/v2/tenants".to_string() + } + + pub fn tenant_path(&self, tenant: &str) -> String { + format!("/admin/v2/tenants/{}", encode_segment(tenant)) + } + + pub fn namespaces_path(&self, tenant: &str) -> String { + format!("/admin/v2/namespaces/{}", encode_segment(tenant)) + } + + pub fn namespace_path(&self, tenant: &str, namespace: &str) -> String { + format!("/admin/v2/namespaces/{}/{}", encode_segment(tenant), encode_segment(namespace)) + } + + pub fn namespace_policies_path(&self, tenant: &str, namespace: &str) -> String { + // GET on the namespace itself returns its policies. + self.namespace_path(tenant, namespace) + } + + pub fn topics_path(&self, tenant: &str, namespace: &str, persistent: bool) -> String { + let domain = if persistent { "persistent" } else { "non-persistent" }; + format!("/admin/v2/{domain}/{}/{}", encode_segment(tenant), encode_segment(namespace)) + } + + pub fn partitioned_topics_path(&self, tenant: &str, namespace: &str, persistent: bool) -> String { + let domain = if persistent { "persistent" } else { "non-persistent" }; + format!("/admin/v2/{domain}/{}/{}/partitioned", encode_segment(tenant), encode_segment(namespace)) + } + + /// Base path for a single topic: `/admin/v2/{domain}/{tenant}/{ns}/{topic}`. + pub fn topic_base_path(&self, domain: &str, topic_path: &str) -> String { + format!("/admin/v2/{domain}/{}", encode_topic_path(topic_path)) + } + + pub fn topic_partitions_path(&self, domain: &str, topic_path: &str) -> String { + format!("/admin/v2/{domain}/{}/partitions", encode_topic_path(topic_path)) + } + + pub fn topic_stats_path(&self, domain: &str, topic_path: &str) -> String { + format!("/admin/v2/{domain}/{}/stats", encode_topic_path(topic_path)) + } + + pub fn topic_partitioned_stats_path(&self, domain: &str, topic_path: &str, per_partition: bool) -> String { + let path = format!("/admin/v2/{domain}/{}/partitioned-stats", encode_topic_path(topic_path)); + if per_partition { + format!("{path}?perPartition=true") + } else { + path + } + } + + pub fn topic_internal_stats_path(&self, domain: &str, topic_path: &str) -> String { + format!("/admin/v2/{domain}/{}/internalStats", encode_topic_path(topic_path)) + } + + pub fn subscriptions_path(&self, domain: &str, topic_path: &str) -> String { + format!("/admin/v2/{domain}/{}/subscriptions", encode_topic_path(topic_path)) + } + + pub fn subscription_path(&self, domain: &str, topic_path: &str, sub: &str) -> String { + format!("/admin/v2/{domain}/{}/subscription/{}", encode_topic_path(topic_path), encode_segment(sub)) + } + + pub fn subscription_reset_cursor_path(&self, domain: &str, topic_path: &str, sub: &str) -> String { + format!("/admin/v2/{domain}/{}/subscription/{}/resetcursor", encode_topic_path(topic_path), encode_segment(sub)) + } + + pub fn subscription_skip_path(&self, domain: &str, topic_path: &str, sub: &str, count: u32) -> String { + format!( + "/admin/v2/{domain}/{}/subscription/{}/skip/{count}", + encode_topic_path(topic_path), + encode_segment(sub) + ) + } + + pub fn subscription_skip_all_path(&self, domain: &str, topic_path: &str, sub: &str) -> String { + format!("/admin/v2/{domain}/{}/subscription/{}/skip_all", encode_topic_path(topic_path), encode_segment(sub)) + } + + pub fn subscription_peek_path(&self, domain: &str, topic_path: &str, sub: &str, position: u32) -> String { + format!( + "/admin/v2/{domain}/{}/subscription/{}/position/{position}", + encode_topic_path(topic_path), + encode_segment(sub) + ) + } + + pub fn subscription_expire_path(&self, domain: &str, topic_path: &str, sub: &str, expire_seconds: i64) -> String { + format!( + "/admin/v2/{domain}/{}/subscription/{}/expireMessages/{expire_seconds}", + encode_topic_path(topic_path), + encode_segment(sub) + ) + } + + pub fn topic_permissions_path(&self, domain: &str, topic_path: &str) -> String { + format!("/admin/v2/{domain}/{}/permissions", encode_topic_path(topic_path)) + } + + pub fn topic_permission_role_path(&self, domain: &str, topic_path: &str, role: &str) -> String { + format!("/admin/v2/{domain}/{}/permissions/{}", encode_topic_path(topic_path), encode_segment(role)) + } + + pub fn namespace_permissions_path(&self, tenant: &str, namespace: &str) -> String { + format!("/admin/v2/namespaces/{}/{}/permissions", encode_segment(tenant), encode_segment(namespace)) + } + + pub fn namespace_permission_role_path(&self, tenant: &str, namespace: &str, role: &str) -> String { + format!( + "/admin/v2/namespaces/{}/{}/permissions/{}", + encode_segment(tenant), + encode_segment(namespace), + encode_segment(role) + ) + } + + pub fn namespace_publish_rate_path(&self, tenant: &str, namespace: &str) -> String { + format!("/admin/v2/namespaces/{}/{}/publishRate", encode_segment(tenant), encode_segment(namespace)) + } + + pub fn namespace_dispatch_rate_path(&self, tenant: &str, namespace: &str) -> String { + format!("/admin/v2/namespaces/{}/{}/dispatchRate", encode_segment(tenant), encode_segment(namespace)) + } + + pub fn namespace_subscribe_rate_path(&self, tenant: &str, namespace: &str) -> String { + format!("/admin/v2/namespaces/{}/{}/subscribeRate", encode_segment(tenant), encode_segment(namespace)) + } + + pub fn namespace_backlog_quota_path(&self, tenant: &str, namespace: &str) -> String { + format!("/admin/v2/namespaces/{}/{}/backlogQuota", encode_segment(tenant), encode_segment(namespace)) + } + + pub fn namespace_retention_path(&self, tenant: &str, namespace: &str) -> String { + format!("/admin/v2/namespaces/{}/{}/retention", encode_segment(tenant), encode_segment(namespace)) + } + + pub fn topic_publish_rate_path(&self, domain: &str, topic_path: &str) -> String { + format!("/admin/v2/{domain}/{}/publishRate", encode_topic_path(topic_path)) + } + + pub fn topic_dispatch_rate_path(&self, domain: &str, topic_path: &str) -> String { + format!("/admin/v2/{domain}/{}/dispatchRate", encode_topic_path(topic_path)) + } + + pub fn topic_backlog_quota_path(&self, domain: &str, topic_path: &str) -> String { + format!("/admin/v2/{domain}/{}/backlogQuota", encode_topic_path(topic_path)) + } + + pub fn topic_retention_path(&self, domain: &str, topic_path: &str) -> String { + format!("/admin/v2/{domain}/{}/retention", encode_topic_path(topic_path)) + } + + // ---- Response parsing (version-tolerant) ---- + + /// Parse the aggregated topic stats payload. Field names are read + /// defensively so minor cross-version renames don't break parsing. + pub fn parse_topic_stats(&self, raw: &serde_json::Value) -> TopicStats { + let f64_of = |key: &str| raw.get(key).and_then(serde_json::Value::as_f64).unwrap_or(0.0); + let i64_of = |key: &str| raw.get(key).and_then(serde_json::Value::as_i64).unwrap_or(0); + + // Backlog appears as `backlogSize` (bytes) in newer stats; some payloads + // expose `msgBacklog` per subscription only. Read both tolerantly. + let backlog_size = raw + .get("backlogSize") + .and_then(serde_json::Value::as_i64) + .or_else(|| raw.get("backlog").and_then(serde_json::Value::as_i64)) + .unwrap_or(0); + + let subscription_count = + raw.get("subscriptions").and_then(serde_json::Value::as_object).map(|m| m.len() as u32).unwrap_or(0); + + let producer_count = raw + .get("publishers") + .and_then(serde_json::Value::as_array) + .map(|a| a.len() as u32) + .or_else(|| raw.get("producerCount").and_then(serde_json::Value::as_u64).map(|v| v as u32)) + .unwrap_or(0); + + TopicStats { + msg_rate_in: f64_of("msgRateIn"), + msg_rate_out: f64_of("msgRateOut"), + msg_throughput_in: f64_of("msgThroughputIn"), + msg_throughput_out: f64_of("msgThroughputOut"), + storage_size: i64_of("storageSize"), + backlog_size, + msg_in_counter: i64_of("msgInCounter"), + msg_out_counter: i64_of("msgOutCounter"), + subscription_count, + producer_count, + raw: raw.clone(), + } + } + + /// Parse the `subscriptions` map from a topic stats payload into a typed list. + pub fn parse_subscriptions(&self, stats_raw: &serde_json::Value) -> Vec { + let mut by_name = std::collections::BTreeMap::new(); + let has_root_subscriptions = append_subscriptions_from_stats(stats_raw, &mut by_name, false); + + if let Some(partitions) = stats_raw.get("partitions").and_then(serde_json::Value::as_object) { + for partition in partitions.values() { + append_subscriptions_from_stats(partition, &mut by_name, has_root_subscriptions); + } + } + + by_name.into_values().collect() + } +} + +fn append_subscriptions_from_stats( + stats_raw: &serde_json::Value, + by_name: &mut std::collections::BTreeMap, + keep_existing_metrics: bool, +) -> bool { + let Some(subs) = stats_raw.get("subscriptions").and_then(serde_json::Value::as_object) else { + return false; + }; + for (name, body) in subs { + merge_subscription(by_name, subscription_from_stats(name, body), keep_existing_metrics); + } + !subs.is_empty() +} + +fn merge_subscription( + by_name: &mut std::collections::BTreeMap, + mut incoming: SubscriptionInfo, + keep_existing_metrics: bool, +) { + let Some(existing) = by_name.get_mut(&incoming.name) else { + by_name.insert(incoming.name.clone(), incoming); + return; + }; + + if existing.sub_type.is_empty() && !incoming.sub_type.is_empty() { + existing.sub_type = incoming.sub_type; + } + if keep_existing_metrics { + existing.msg_backlog = existing.msg_backlog.max(incoming.msg_backlog); + existing.msg_rate_out = existing.msg_rate_out.max(incoming.msg_rate_out); + existing.msg_throughput_out = existing.msg_throughput_out.max(incoming.msg_throughput_out); + } else { + existing.msg_backlog = existing.msg_backlog.saturating_add(incoming.msg_backlog); + existing.msg_rate_out += incoming.msg_rate_out; + existing.msg_throughput_out += incoming.msg_throughput_out; + } + existing.consumers.append(&mut incoming.consumers); +} + +fn subscription_from_stats(name: &str, body: &serde_json::Value) -> SubscriptionInfo { + let consumers = body + .get("consumers") + .and_then(serde_json::Value::as_array) + .map(|arr| arr.iter().map(parse_consumer).collect()) + .unwrap_or_default(); + SubscriptionInfo { + name: name.to_string(), + sub_type: body.get("type").and_then(serde_json::Value::as_str).unwrap_or("").to_string(), + msg_backlog: body.get("msgBacklog").and_then(serde_json::Value::as_i64).unwrap_or(0), + msg_rate_out: body.get("msgRateOut").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + msg_throughput_out: body.get("msgThroughputOut").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + consumers, + } +} + +fn parse_consumer(v: &serde_json::Value) -> super::super::types::ConsumerInfo { + super::super::types::ConsumerInfo { + consumer_name: v.get("consumerName").and_then(serde_json::Value::as_str).unwrap_or("").to_string(), + msg_rate_out: v.get("msgRateOut").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + msg_throughput_out: v.get("msgThroughputOut").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + available_permits: v.get("availablePermits").and_then(serde_json::Value::as_i64).unwrap_or(0), + address: v.get("address").and_then(serde_json::Value::as_str).unwrap_or("").to_string(), + client_version: v.get("clientVersion").and_then(serde_json::Value::as_str).unwrap_or("").to_string(), + } +} + +fn encode_segment(segment: &str) -> String { + utf8_percent_encode(segment, NON_ALPHANUMERIC).to_string() +} + +fn encode_topic_path(topic_path: &str) -> String { + let mut parts = topic_path.splitn(3, '/'); + let tenant = parts.next(); + let namespace = parts.next(); + let topic = parts.next(); + match (tenant, namespace, topic) { + (Some(tenant), Some(namespace), Some(topic)) => { + format!("{}/{}/{}", encode_segment(tenant), encode_segment(namespace), encode_segment(topic)) + } + _ => topic_path.split('/').map(encode_segment).collect::>().join("/"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_plain_version_strings() { + assert_eq!(PulsarApiProfile::parse_version("3.1.2"), Some(PulsarVersion::V3_1)); + assert_eq!(PulsarApiProfile::parse_version("3.0.7"), Some(PulsarVersion::V3_1)); + assert_eq!(PulsarApiProfile::parse_version(" 3.1.0 "), Some(PulsarVersion::V3_1)); + } + + #[test] + fn parses_verbose_version_strings() { + assert_eq!(PulsarApiProfile::parse_version("Pulsar version 3.1.2 (rev abc)"), Some(PulsarVersion::V3_1)); + } + + #[test] + fn unknown_major_returns_none() { + assert_eq!(PulsarApiProfile::parse_version("4.0.1"), None); + assert_eq!(PulsarApiProfile::parse_version("not-a-version"), None); + } + + #[test] + fn baseline_profile_is_fallback_3_1() { + let p = PulsarApiProfile::default_baseline(); + assert_eq!(p.version(), PulsarVersion::V3_1); + assert_eq!(p.detection(), VersionDetection::Fallback); + assert!(p.capabilities().supports_tenants); + assert!(!p.capabilities().supports_token_management); + } + + #[test] + fn parses_topic_stats_fields() { + let raw = serde_json::json!({ + "msgRateIn": 12.5, + "msgRateOut": 7.0, + "storageSize": 4096, + "backlogSize": 128, + "publishers": [{"producerName": "p1"}], + "subscriptions": { + "sub-a": {"msgBacklog": 10, "msgRateOut": 1.0, "type": "Shared", "consumers": []}, + "sub-b": {"msgBacklog": 0, "type": "Exclusive", "consumers": []} + } + }); + let profile = PulsarApiProfile::default_baseline(); + let stats = profile.parse_topic_stats(&raw); + assert_eq!(stats.msg_rate_in, 12.5); + assert_eq!(stats.storage_size, 4096); + assert_eq!(stats.backlog_size, 128); + assert_eq!(stats.subscription_count, 2); + assert_eq!(stats.producer_count, 1); + + let subs = profile.parse_subscriptions(&raw); + assert_eq!(subs.len(), 2); + } + + #[test] + fn parses_partitioned_stats_subscriptions() { + let raw = serde_json::json!({ + "partitions": { + "persistent://public/default/orders-partition-0": { + "subscriptions": { + "sub-a": { + "msgBacklog": 3, + "msgRateOut": 1.5, + "msgThroughputOut": 128.0, + "type": "Shared", + "consumers": [{ "consumerName": "c0", "address": "/127.0.0.1:1000" }] + } + } + }, + "persistent://public/default/orders-partition-1": { + "subscriptions": { + "sub-a": { + "msgBacklog": 5, + "msgRateOut": 2.0, + "msgThroughputOut": 256.0, + "type": "Shared", + "consumers": [{ "consumerName": "c1", "address": "/127.0.0.1:1001" }] + } + } + } + } + }); + let profile = PulsarApiProfile::default_baseline(); + let subs = profile.parse_subscriptions(&raw); + assert_eq!(subs.len(), 1); + assert_eq!(subs[0].name, "sub-a"); + assert_eq!(subs[0].sub_type, "Shared"); + assert_eq!(subs[0].msg_backlog, 8); + assert_eq!(subs[0].msg_rate_out, 3.5); + assert_eq!(subs[0].msg_throughput_out, 384.0); + assert_eq!(subs[0].consumers.len(), 2); + } + + #[test] + fn encodes_each_admin_path_segment() { + let profile = PulsarApiProfile::default_baseline(); + assert_eq!(profile.tenant_path("tenant with/slash?x"), "/admin/v2/tenants/tenant%20with%2Fslash%3Fx"); + assert_eq!(profile.namespace_path("public", "default space"), "/admin/v2/namespaces/public/default%20space"); + assert_eq!( + profile.topic_base_path("persistent", "public/default/orders/a?b"), + "/admin/v2/persistent/public/default/orders%2Fa%3Fb" + ); + assert_eq!( + profile.subscription_path("persistent", "public/default/orders/a?b", "sub/role #1"), + "/admin/v2/persistent/public/default/orders%2Fa%3Fb/subscription/sub%2Frole%20%231" + ); + assert_eq!( + profile.topic_permission_role_path("persistent", "public/default/orders", "role/a?b"), + "/admin/v2/persistent/public/default/orders/permissions/role%2Fa%3Fb" + ); + } + + #[test] + fn partitioned_stats_path_encodes_topic_and_requests_partition_detail() { + let profile = PulsarApiProfile::default_baseline(); + assert_eq!( + profile.topic_partitioned_stats_path("persistent", "b2b/ec-product-service/goods_status_change", true), + "/admin/v2/persistent/b2b/ec%2Dproduct%2Dservice/goods%5Fstatus%5Fchange/partitioned-stats?perPartition=true" + ); + assert_eq!( + profile.topic_partitioned_stats_path("persistent", "public/default/orders/a?b", true), + "/admin/v2/persistent/public/default/orders%2Fa%3Fb/partitioned-stats?perPartition=true" + ); + } + + #[test] + fn peek_path_encodes_topic_and_subscription_segments() { + let profile = PulsarApiProfile::default_baseline(); + assert_eq!( + profile.subscription_peek_path("persistent", "public/default/orders/a?b", "sub/role #1", 3), + "/admin/v2/persistent/public/default/orders%2Fa%3Fb/subscription/sub%2Frole%20%231/position/3" + ); + } +} diff --git a/crates/dbx-core/src/mq/auth.rs b/crates/dbx-core/src/mq/auth.rs new file mode 100644 index 000000000..124041786 --- /dev/null +++ b/crates/dbx-core/src/mq/auth.rs @@ -0,0 +1,259 @@ +//! Authentication for message queue admin connections. Injects credentials +//! into outgoing `reqwest` requests, with token caching for OAuth2 +//! client-credentials so we don't exchange a token on every call. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; + +use super::util::truncate; + +/// Authentication method for an MQ admin connection. +/// +/// Sensitive fields (token / password / secret) are stored through the +/// project's encrypted connection-secrets / keychain path and are never logged. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum MqAuth { + None, + /// Pulsar JWT bearer token. + Token { + token: String, + }, + /// HTTP Basic auth. + Basic { + username: String, + password: String, + }, + /// Arbitrary API key header, e.g. `Authorization: ` or a custom header. + ApiKey { + header: String, + value: String, + }, + /// OAuth2 client-credentials flow (Pulsar's `oauth2` auth plugin). + OAuth2 { + issuer_url: String, + client_id: String, + client_secret: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + audience: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + scope: Option, + }, +} + +impl Default for MqAuth { + fn default() -> Self { + MqAuth::None + } +} + +/// Caches an OAuth2 access token with its expiry so repeated requests reuse it. +/// Also tracks in-flight token requests to prevent concurrent fetches. +#[derive(Debug, Default, Clone)] +pub struct TokenCache { + inner: Arc>, +} + +#[derive(Debug, Default)] +struct TokenCacheState { + cached: Option, + /// Tracks whether a token fetch is currently in progress. Subsequent requests + /// will wait on this shared future instead of issuing duplicate OAuth2 requests. + #[allow(clippy::type_complexity)] + in_flight: Option>, +} + +#[derive(Debug, Clone)] +struct CachedToken { + token: String, + expires_at: Instant, +} + +/// Refresh the cached token slightly before it actually expires. +const TOKEN_EXPIRY_SKEW: Duration = Duration::from_secs(30); + +#[derive(Deserialize)] +struct OAuth2TokenResponse { + access_token: String, + #[serde(default)] + expires_in: Option, +} + +impl MqAuth { + /// Apply this auth method to a request builder, fetching/refreshing an + /// OAuth2 token through `http`/`cache` if necessary. + pub async fn apply( + &self, + req: reqwest::RequestBuilder, + http: &reqwest::Client, + cache: &TokenCache, + ) -> Result { + match self { + MqAuth::None => Ok(req), + MqAuth::Token { token } => Ok(req.bearer_auth(token)), + MqAuth::Basic { username, password } => Ok(req.basic_auth(username, Some(password))), + MqAuth::ApiKey { header, value } => Ok(req.header(header.as_str(), value.as_str())), + MqAuth::OAuth2 { .. } => { + let token = self.fetch_oauth2_token(http, cache).await?; + Ok(req.bearer_auth(token)) + } + } + } + + async fn fetch_oauth2_token(&self, http: &reqwest::Client, cache: &TokenCache) -> Result { + let MqAuth::OAuth2 { issuer_url, client_id, client_secret, audience, scope } = self else { + return Err("not an OAuth2 auth method".to_string()); + }; + + // Fast path: reuse a still-valid cached token. + let notify = { + let mut guard = cache.inner.lock().await; + if let Some(cached) = &guard.cached { + if cached.expires_at > Instant::now() { + return Ok(cached.token.clone()); + } + } + + // Check if another task is already fetching a token + if let Some(existing_notify) = &guard.in_flight { + let notify = existing_notify.clone(); + drop(guard); + // Wait for the in-flight request to complete + notify.notified().await; + // Re-check the cache after waiting + let guard = cache.inner.lock().await; + if let Some(cached) = &guard.cached { + if cached.expires_at > Instant::now() { + return Ok(cached.token.clone()); + } + } + // Token still expired after waiting, we'll fetch ourselves + return Err("Token fetch by another task failed or token still expired".to_string()); + } + + // Mark that we're now fetching + let notify = Arc::new(tokio::sync::Notify::new()); + guard.in_flight = Some(notify.clone()); + notify + }; + + // Fetch the token (without holding the lock) + let result = self.do_oauth2_token_fetch(http, issuer_url, client_id, client_secret, audience, scope).await; + + // Update cache and clear in-flight flag + { + let mut guard = cache.inner.lock().await; + guard.in_flight = None; + + match result { + Ok((token, ttl)) => { + let expires_at = Instant::now() + Duration::from_secs(ttl).saturating_sub(TOKEN_EXPIRY_SKEW); + guard.cached = Some(CachedToken { token: token.clone(), expires_at }); + notify.notify_waiters(); + Ok(token) + } + Err(e) => { + notify.notify_waiters(); + Err(e) + } + } + } + } + + async fn do_oauth2_token_fetch( + &self, + http: &reqwest::Client, + issuer_url: &str, + client_id: &str, + client_secret: &str, + audience: &Option, + scope: &Option, + ) -> Result<(String, u64), String> { + let mut form = vec![ + ("grant_type", "client_credentials".to_string()), + ("client_id", client_id.to_string()), + ("client_secret", client_secret.to_string()), + ]; + if let Some(audience) = audience { + form.push(("audience", audience.clone())); + } + if let Some(scope) = scope { + form.push(("scope", scope.clone())); + } + + let resp = + http.post(issuer_url).form(&form).send().await.map_err(|e| format!("OAuth2 token request failed: {e}"))?; + + let status = resp.status(); + if !status.is_success() { + let body = resp.text().await.unwrap_or_default(); + let detail = oauth2_error_detail(&body).unwrap_or_else(|| truncate(&body, 300)); + return Err(format!("OAuth2 token endpoint returned {status}: {detail}")); + } + + let parsed: OAuth2TokenResponse = + resp.json().await.map_err(|e| format!("Failed to parse OAuth2 token response: {e}"))?; + + let ttl = parsed.expires_in.unwrap_or(3600); + Ok((parsed.access_token, ttl)) + } +} + +/// Extract a human-readable OAuth2 error detail from a JSON error response. +/// Returns `None` when the body is not a valid OAuth2 error payload. +fn oauth2_error_detail(body: &str) -> Option { + let value: serde_json::Value = serde_json::from_str(body).ok()?; + let error = value.get("error")?.as_str()?; + let description = value.get("error_description").and_then(serde_json::Value::as_str); + Some(match description { + Some(desc) => format!("{error}: {desc}"), + None => error.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn auth_default_is_none() { + assert!(matches!(MqAuth::default(), MqAuth::None)); + } + + #[test] + fn token_auth_serde_roundtrip() { + let auth = MqAuth::Token { token: "abc".to_string() }; + let json = serde_json::to_string(&auth).expect("should serialize MqAuth"); + assert!(json.contains("\"kind\":\"token\"")); + let back: MqAuth = serde_json::from_str(&json).expect("should deserialize MqAuth"); + assert!(matches!(back, MqAuth::Token { token } if token == "abc")); + } + + #[test] + fn oauth2_optional_fields_skipped_when_absent() { + let auth = MqAuth::OAuth2 { + issuer_url: "https://issuer/token".to_string(), + client_id: "id".to_string(), + client_secret: "secret".to_string(), + audience: None, + scope: None, + }; + let json = serde_json::to_string(&auth).expect("should serialize OAuth2 auth"); + assert!(!json.contains("audience")); + assert!(!json.contains("scope")); + } + + #[test] + fn oauth2_error_detail_parses_standard_error() { + let body = r#"{"error":"invalid_client","error_description":"Client authentication failed"}"#; + assert_eq!(oauth2_error_detail(body).as_deref(), Some("invalid_client: Client authentication failed")); + } + + #[test] + fn oauth2_error_detail_returns_none_for_non_json() { + assert!(oauth2_error_detail("not json").is_none()); + } +} diff --git a/crates/dbx-core/src/mq/config.rs b/crates/dbx-core/src/mq/config.rs new file mode 100644 index 000000000..ddf4d0803 --- /dev/null +++ b/crates/dbx-core/src/mq/config.rs @@ -0,0 +1,183 @@ +//! Parses the message queue admin configuration out of a `ConnectionConfig`. +//! +//! MQ admin connections reuse the generic `external_config` extension slot on +//! `ConnectionConfig` rather than adding top-level fields, keeping the 50+ +//! database-type connection model untouched. + +use serde::{Deserialize, Serialize}; + +use crate::models::connection::ConnectionConfig; +use crate::mq::auth::MqAuth; +use crate::mq::types::{MqSystemKind, MqTokenSigningConfig}; + +/// Runtime TCP endpoint override for MQ admin requests. +/// +/// The public admin URL remains unchanged so TLS hostname verification, SNI and +/// the HTTP Host header continue to target the broker name. The HTTP client uses +/// this endpoint only for the underlying TCP connection, e.g. after an SSH/proxy +/// tunnel has mapped the broker to a local port. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MqConnectOverride { + pub host: String, + pub port: u16, +} + +/// Configuration for an MQ admin connection, decoded from +/// `ConnectionConfig.external_config`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MqAdminConfig { + pub system_kind: MqSystemKind, + /// Admin REST base URL, e.g. `http://broker:8080`. + pub admin_url: String, + #[serde(default)] + pub auth: MqAuth, + /// Skip TLS certificate verification (self-signed clusters only). + #[serde(default)] + pub tls_skip_verify: bool, + /// Manually pin a server version (e.g. `3.1`), skipping auto-detection for + /// environments where the version endpoint is blocked by a gateway. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pinned_version: Option, + /// Optional local JWT signing configuration. This is used only by dbx to + /// issue Pulsar client tokens; the signing key itself is stored through the + /// connection secret path, not in plain connection JSON. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token_signing: Option, + /// Runtime-only TCP endpoint override used by transport layers. + #[serde(skip)] + pub connect_override: Option, + /// System-specific extension fields (e.g. Kafka bootstrap servers). + #[serde(default, skip_serializing_if = "serde_json::Value::is_null")] + pub extra: serde_json::Value, +} + +impl MqAdminConfig { + /// Decode the MQ admin config from a connection's `external_config` slot. + pub fn from_connection(cfg: &ConnectionConfig) -> Result { + let raw = cfg + .external_config + .as_ref() + .ok_or("This connection has no message queue admin configuration (external_config is empty)")?; + let mut parsed: MqAdminConfig = serde_json::from_value(raw.clone()) + .map_err(|e| format!("Failed to parse message queue admin config: {e}"))?; + parsed.admin_url = parsed.admin_url.trim().to_string(); + if parsed.admin_url.is_empty() { + return Err("Message queue admin URL is empty".to_string()); + } + Ok(parsed) + } + + pub fn token_signing_configured(&self) -> bool { + self.token_signing.as_ref().is_some_and(MqTokenSigningConfig::is_configured) + } + + pub fn with_admin_endpoint(mut self, host: &str, port: u16) -> Result { + self.admin_url = admin_url_with_endpoint(&self.admin_url, host, port)?; + Ok(self) + } + + pub fn with_connect_override(mut self, host: &str, port: u16) -> Self { + self.connect_override = Some(MqConnectOverride { host: host.to_string(), port }); + self + } +} + +pub fn admin_url_with_endpoint(admin_url: &str, host: &str, port: u16) -> Result { + let mut url = reqwest::Url::parse(admin_url).map_err(|e| format!("MQ Admin URL is invalid: {e}"))?; + url.set_host(Some(host)).map_err(|_| format!("MQ Admin URL host is invalid: {host}"))?; + url.set_port(Some(port)).map_err(|_| format!("MQ Admin URL port is invalid: {port}"))?; + Ok(url.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn connection_with_external(value: serde_json::Value) -> ConnectionConfig { + let mut cfg = ConnectionConfig { + id: "c1".to_string(), + name: "mq".to_string(), + db_type: crate::models::connection::DatabaseType::MessageQueue, + driver_profile: None, + driver_label: None, + url_params: None, + host: String::new(), + port: 0, + username: String::new(), + password: String::new(), + database: None, + visible_databases: None, + attached_databases: Vec::new(), + color: None, + transport_layers: Vec::new(), + connect_timeout_secs: 5, + query_timeout_secs: 30, + idle_timeout_secs: 60, + keepalive_interval_secs: crate::models::connection::default_keepalive_interval_secs(), + ssl: false, + ca_cert_path: String::new(), + client_cert_path: String::new(), + client_key_path: String::new(), + sysdba: false, + oracle_connection_type: None, + connection_string: None, + redis_connection_mode: None, + redis_sentinel_master: String::new(), + redis_sentinel_nodes: String::new(), + redis_sentinel_username: String::new(), + redis_sentinel_password: String::new(), + redis_sentinel_tls: false, + redis_cluster_nodes: String::new(), + redis_key_separator: String::new(), + etcd_endpoints: String::new(), + gbase_server: String::new(), + external_config: Some(value), + jdbc_driver_class: None, + jdbc_driver_paths: Vec::new(), + one_time: false, + read_only: false, + }; + cfg.redis_key_separator = ":".to_string(); + cfg + } + + #[test] + fn parses_pulsar_config() { + let cfg = connection_with_external(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://localhost:8080", + "auth": { "kind": "none" } + })); + let mqc = MqAdminConfig::from_connection(&cfg).expect("should parse valid Pulsar config"); + assert_eq!(mqc.system_kind, MqSystemKind::Pulsar); + assert_eq!(mqc.admin_url, "http://localhost:8080"); + assert!(matches!(mqc.auth, MqAuth::None)); + } + + #[test] + fn errors_when_external_config_missing() { + let mut cfg = connection_with_external(serde_json::Value::Null); + cfg.external_config = None; + assert!(MqAdminConfig::from_connection(&cfg).is_err()); + } + + #[test] + fn errors_on_empty_admin_url() { + let cfg = connection_with_external(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": " " + })); + assert!(MqAdminConfig::from_connection(&cfg).is_err()); + } + + #[test] + fn admin_url_with_endpoint_preserves_scheme_path_and_query() { + let rewritten = + admin_url_with_endpoint("https://broker.internal:8443/pulsar-admin?tenant=public", "127.0.0.1", 49152) + .expect("should rewrite admin URL with endpoint"); + + assert_eq!(rewritten, "https://127.0.0.1:49152/pulsar-admin?tenant=public"); + } +} diff --git a/crates/dbx-core/src/mq/mod.rs b/crates/dbx-core/src/mq/mod.rs new file mode 100644 index 000000000..864a34a37 --- /dev/null +++ b/crates/dbx-core/src/mq/mod.rs @@ -0,0 +1,116 @@ +//! Message queue admin console support. +//! +//! Provides a pluggable, port-adapter abstraction for managing message queue +//! systems (Apache Pulsar today; Kafka / RocketMQ are reserved). The module is +//! gated behind the `mq-admin` Cargo feature so builds that don't need it pay +//! nothing. +//! +//! Architecture mirrors the existing `agent_kv` pattern: business logic lives in +//! `service::*_core` functions shared by the desktop command layer and the web +//! route layer; this module owns the trait, the typed model, and the registry +//! that caches one adapter per connection. + +pub mod adapters; +pub mod auth; +pub mod config; +pub mod port; +pub mod service; +pub mod token; +pub mod types; +pub(crate) mod util; + +use std::collections::HashMap; +use std::sync::Arc; + +use tokio::sync::{Mutex, RwLock}; + +use crate::models::connection::ConnectionConfig; +use crate::mq::adapters::pulsar::PulsarAdmin; +use crate::mq::config::MqAdminConfig; +use crate::mq::port::MessageQueueAdmin; +use crate::mq::types::MqSystemKind as MqSystemKindInternal; + +pub use crate::mq::auth::MqAuth; +pub use crate::mq::config::MqAdminConfig as MqConfig; +pub use crate::mq::types::*; + +/// Caches one live admin adapter per connection id. Adapters are built lazily on +/// first use and dropped when the connection is closed. +/// +/// Each connection has its own build-lock so concurrent first-use requests for +/// the same connection block until the first builder finishes, rather than both +/// racing to construct an adapter. +#[derive(Default)] +pub struct MqAdminRegistry { + instances: RwLock>>, + build_locks: RwLock>>>, +} + +impl MqAdminRegistry { + pub fn new() -> Self { + Self { instances: RwLock::new(HashMap::new()), build_locks: RwLock::new(HashMap::new()) } + } + + /// Return the cached adapter for this connection, building it from the + /// connection's `external_config` if not already present. + pub async fn get_or_build(&self, cfg: &ConnectionConfig) -> Result, String> { + let mqc = MqAdminConfig::from_connection(cfg)?; + self.get_or_build_config(&cfg.id, mqc).await + } + + pub async fn get_or_build_config( + &self, + connection_id: &str, + mqc: MqAdminConfig, + ) -> Result, String> { + // Fast path: return the cached adapter. + if let Some(adapter) = self.instances.read().await.get(connection_id) { + return Ok(adapter.clone()); + } + + // Slow path: acquire a per-connection build lock so only one task + // constructs the adapter at a time. + let lock = { + let mut locks = self.build_locks.write().await; + locks.entry(connection_id.to_string()).or_insert_with(|| Arc::new(Mutex::new(()))).clone() + }; + let _guard = lock.lock().await; + + // Another task may have built it while we were waiting for the lock. + if let Some(adapter) = self.instances.read().await.get(connection_id) { + return Ok(adapter.clone()); + } + + let adapter = build_adapter(mqc).await?; + self.instances.write().await.insert(connection_id.to_string(), adapter.clone()); + Ok(adapter) + } + + /// Drop the cached adapter for a connection (called on disconnect). + pub async fn drop_connection(&self, connection_id: &str) { + self.instances.write().await.remove(connection_id); + self.build_locks.write().await.remove(connection_id); + } + + /// Build a fresh adapter without caching it — used for connection tests + /// where we don't want to retain state. + pub async fn build_transient(&self, cfg: &ConnectionConfig) -> Result, String> { + let mqc = MqAdminConfig::from_connection(cfg)?; + self.build_transient_config(mqc).await + } + + pub async fn build_transient_config(&self, mqc: MqAdminConfig) -> Result, String> { + build_adapter(mqc).await + } +} + +async fn build_adapter(mqc: MqAdminConfig) -> Result, String> { + match mqc.system_kind { + MqSystemKindInternal::Pulsar => { + let adapter = PulsarAdmin::new(mqc).await?; + Ok(Arc::new(adapter)) + } + MqSystemKindInternal::Kafka => Err("Kafka admin is not yet implemented".to_string()), + MqSystemKindInternal::RocketMq => Err("RocketMQ admin is not yet implemented".to_string()), + } +} diff --git a/crates/dbx-core/src/mq/port.rs b/crates/dbx-core/src/mq/port.rs new file mode 100644 index 000000000..627c1371e --- /dev/null +++ b/crates/dbx-core/src/mq/port.rs @@ -0,0 +1,77 @@ +//! The message queue admin port — the abstraction all MQ systems implement. +//! +//! Commands and routes program against this trait only; concrete systems +//! (Pulsar today, Kafka / RocketMQ later) live in `adapters/`. + +use async_trait::async_trait; + +use crate::mq::types::*; + +/// Unified management interface for message queue systems. +#[async_trait] +pub trait MessageQueueAdmin: Send + Sync { + /// Capability flags so the frontend can show/hide functionality. + fn capabilities(&self) -> MqCapabilities; + + fn system_kind(&self) -> MqSystemKind; + + /// Connectivity test; returns cluster/version info. + async fn test_connection(&self) -> Result; + + // ---- Tenants ---- + async fn list_tenants(&self) -> Result, String>; + async fn get_tenant(&self, name: &str) -> Result; + async fn create_tenant(&self, name: &str, cfg: TenantConfig) -> Result<(), String>; + async fn update_tenant(&self, name: &str, cfg: TenantConfig) -> Result<(), String>; + async fn delete_tenant(&self, name: &str, force: bool) -> Result<(), String>; + + // ---- Namespaces ---- + async fn list_namespaces(&self, tenant: &str) -> Result, String>; + async fn create_namespace(&self, ns: &NamespaceRef, cfg: NamespaceConfig) -> Result<(), String>; + async fn delete_namespace(&self, ns: &NamespaceRef, force: bool) -> Result<(), String>; + async fn get_namespace_policies(&self, ns: &NamespaceRef) -> Result; + + // ---- Topics ---- + async fn list_topics(&self, ns: &NamespaceRef, opts: ListTopicsOpts) -> Result, String>; + async fn create_topic(&self, topic: &TopicRef, partitions: Option) -> Result<(), String>; + async fn delete_topic(&self, topic: &TopicRef, force: bool) -> Result<(), String>; + async fn update_partitions(&self, topic: &TopicRef, partitions: u32) -> Result<(), String>; + async fn get_topic_stats(&self, topic: &TopicRef) -> Result; + async fn get_topic_internal_stats(&self, topic: &TopicRef) -> Result; + + // ---- Subscriptions ---- + async fn list_subscriptions(&self, topic: &TopicRef) -> Result, String>; + async fn create_subscription(&self, topic: &TopicRef, sub: &str, pos: ResetPosition) -> Result<(), String>; + async fn delete_subscription(&self, topic: &TopicRef, sub: &str, force: bool) -> Result<(), String>; + async fn skip_messages(&self, topic: &TopicRef, sub: &str, count: SkipCount) -> Result<(), String>; + async fn reset_cursor(&self, topic: &TopicRef, sub: &str, pos: ResetPosition) -> Result<(), String>; + async fn clear_backlog(&self, topic: &TopicRef, sub: &str) -> Result<(), String>; + async fn peek_messages(&self, topic: &TopicRef, sub: &str, count: u32) -> Result, String>; + async fn expire_messages(&self, topic: &TopicRef, sub: &str, expire_seconds: i64) -> Result<(), String>; + + // ---- Producers / consumers (runtime, read from stats) ---- + async fn list_producers(&self, topic: &TopicRef) -> Result, String>; + async fn list_consumers(&self, topic: &TopicRef, sub: &str) -> Result, String>; + /// Unload a topic (Pulsar has no per-connection kill; unload is the closest). + async fn unload_topic(&self, topic: &TopicRef) -> Result<(), String>; + + // ---- Rate limits / quotas / retention ---- + async fn set_publish_rate(&self, scope: &PolicyScope, rate: PublishRate) -> Result<(), String>; + async fn set_dispatch_rate(&self, scope: &PolicyScope, rate: DispatchRate) -> Result<(), String>; + async fn set_subscribe_rate(&self, scope: &PolicyScope, rate: SubscribeRate) -> Result<(), String>; + async fn set_backlog_quota(&self, scope: &PolicyScope, quota: BacklogQuota) -> Result<(), String>; + async fn set_retention(&self, scope: &PolicyScope, retention: RetentionPolicy) -> Result<(), String>; + async fn get_effective_policies(&self, scope: &PolicyScope) -> Result; + + // ---- Permissions (who can produce / consume) ---- + async fn grant_permission(&self, scope: &PolicyScope, role: &str, actions: Vec) -> Result<(), String>; + async fn revoke_permission(&self, scope: &PolicyScope, role: &str) -> Result<(), String>; + async fn list_permissions(&self, scope: &PolicyScope) -> Result; + + // ---- Monitoring ---- + async fn get_backlog(&self, topic: &TopicRef, sub: Option<&str>) -> Result; + + /// Escape hatch: proxy an arbitrary admin REST call. Covers any endpoint the + /// typed methods do not. + async fn raw_request(&self, req: MqRawRequest) -> Result; +} diff --git a/crates/dbx-core/src/mq/service.rs b/crates/dbx-core/src/mq/service.rs new file mode 100644 index 000000000..ce1d0c542 --- /dev/null +++ b/crates/dbx-core/src/mq/service.rs @@ -0,0 +1,661 @@ +//! Service-layer functions for message queue admin operations. These `*_core` +//! functions are shared by both the desktop command layer +//! (`src-tauri/src/commands/mq_cmd.rs`) and the web route layer +//! (`crates/dbx-web/src/routes/mq.rs`), keeping the business logic unified. +//! +//! Mirrors the pattern used by `agent_kv::*_core`. + +use crate::connection::AppState; +use crate::mq::config::MqAdminConfig; +use crate::mq::token::sign_pulsar_token; +use crate::mq::types::*; +use chrono::{TimeZone, Utc}; +use uuid::Uuid; + +const MAX_PEEK_MESSAGES: u32 = 100; + +/// Test connectivity to the message queue admin endpoint. Builds a transient +/// adapter (not cached) to avoid polluting the registry on failed attempts. +pub async fn mq_test_connection_core(state: &AppState, conn_id: &str) -> Result { + let cfg = state.configs.read().await.get(conn_id).cloned().ok_or("Connection not found")?; + let mqc = state.mq_admin_config_for_connection(conn_id, &cfg).await?; + let adapter = state.mq_registry.build_transient_config(mqc).await?; + adapter.test_connection().await +} + +// ---- Tenants ---- + +pub async fn mq_list_tenants_core(state: &AppState, conn_id: &str) -> Result, String> { + let adapter = get_adapter(state, conn_id).await?; + adapter.list_tenants().await +} + +pub async fn mq_get_tenant_core(state: &AppState, conn_id: &str, name: &str) -> Result { + let adapter = get_adapter(state, conn_id).await?; + adapter.get_tenant(name).await +} + +pub async fn mq_create_tenant_core( + state: &AppState, + conn_id: &str, + name: &str, + cfg: TenantConfig, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Create tenant").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.create_tenant(name, cfg).await +} + +pub async fn mq_update_tenant_core( + state: &AppState, + conn_id: &str, + name: &str, + cfg: TenantConfig, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Update tenant").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.update_tenant(name, cfg).await +} + +pub async fn mq_delete_tenant_core(state: &AppState, conn_id: &str, name: &str, force: bool) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Delete tenant").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.delete_tenant(name, force).await +} + +// ---- Namespaces ---- + +pub async fn mq_list_namespaces_core( + state: &AppState, + conn_id: &str, + tenant: &str, +) -> Result, String> { + let adapter = get_adapter(state, conn_id).await?; + adapter.list_namespaces(tenant).await +} + +pub async fn mq_create_namespace_core( + state: &AppState, + conn_id: &str, + ns: NamespaceRef, + cfg: NamespaceConfig, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Create namespace").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.create_namespace(&ns, cfg).await +} + +pub async fn mq_delete_namespace_core( + state: &AppState, + conn_id: &str, + ns: NamespaceRef, + force: bool, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Delete namespace").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.delete_namespace(&ns, force).await +} + +pub async fn mq_get_namespace_policies_core( + state: &AppState, + conn_id: &str, + ns: NamespaceRef, +) -> Result { + let adapter = get_adapter(state, conn_id).await?; + adapter.get_namespace_policies(&ns).await +} + +// ---- Topics ---- + +pub async fn mq_list_topics_core( + state: &AppState, + conn_id: &str, + ns: NamespaceRef, + opts: ListTopicsOpts, +) -> Result, String> { + let adapter = get_adapter(state, conn_id).await?; + adapter.list_topics(&ns, opts).await +} + +pub async fn mq_create_topic_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + partitions: Option, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Create topic").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.create_topic(&topic, partitions).await +} + +pub async fn mq_delete_topic_core(state: &AppState, conn_id: &str, topic: TopicRef, force: bool) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Delete topic").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.delete_topic(&topic, force).await +} + +pub async fn mq_update_partitions_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + partitions: u32, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Update partitions").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.update_partitions(&topic, partitions).await +} + +pub async fn mq_get_topic_stats_core(state: &AppState, conn_id: &str, topic: TopicRef) -> Result { + let adapter = get_adapter(state, conn_id).await?; + adapter.get_topic_stats(&topic).await +} + +pub async fn mq_get_topic_internal_stats_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, +) -> Result { + let adapter = get_adapter(state, conn_id).await?; + adapter.get_topic_internal_stats(&topic).await +} + +// ---- Subscriptions ---- + +pub async fn mq_list_subscriptions_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, +) -> Result, String> { + let adapter = get_adapter(state, conn_id).await?; + adapter.list_subscriptions(&topic).await +} + +pub async fn mq_create_subscription_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + sub: String, + pos: ResetPosition, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Create subscription").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.create_subscription(&topic, &sub, pos).await +} + +pub async fn mq_delete_subscription_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + sub: String, + force: bool, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Delete subscription").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.delete_subscription(&topic, &sub, force).await +} + +pub async fn mq_skip_messages_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + sub: String, + count: SkipCount, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Skip messages").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.skip_messages(&topic, &sub, count).await +} + +pub async fn mq_reset_cursor_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + sub: String, + pos: ResetPosition, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Reset cursor").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.reset_cursor(&topic, &sub, pos).await +} + +pub async fn mq_clear_backlog_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + sub: String, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Clear backlog").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.clear_backlog(&topic, &sub).await +} + +pub async fn mq_peek_messages_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + sub: String, + count: u32, +) -> Result, String> { + if count == 0 { + return Ok(Vec::new()); + } + if count > MAX_PEEK_MESSAGES { + return Err(format!("Peek message count must be between 1 and {MAX_PEEK_MESSAGES}")); + } + let adapter = get_adapter(state, conn_id).await?; + adapter.peek_messages(&topic, &sub, count).await +} + +pub async fn mq_expire_messages_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + sub: String, + expire_seconds: i64, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Expire messages").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.expire_messages(&topic, &sub, expire_seconds).await +} + +// ---- Producers / consumers ---- + +pub async fn mq_list_producers_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, +) -> Result, String> { + let adapter = get_adapter(state, conn_id).await?; + adapter.list_producers(&topic).await +} + +pub async fn mq_list_consumers_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + sub: String, +) -> Result, String> { + let adapter = get_adapter(state, conn_id).await?; + adapter.list_consumers(&topic, &sub).await +} + +pub async fn mq_unload_topic_core(state: &AppState, conn_id: &str, topic: TopicRef) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Unload topic").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.unload_topic(&topic).await +} + +// ---- Rate limits / quotas / retention ---- + +pub async fn mq_set_publish_rate_core( + state: &AppState, + conn_id: &str, + scope: PolicyScope, + rate: PublishRate, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Set publish rate").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.set_publish_rate(&scope, rate).await +} + +pub async fn mq_set_dispatch_rate_core( + state: &AppState, + conn_id: &str, + scope: PolicyScope, + rate: DispatchRate, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Set dispatch rate").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.set_dispatch_rate(&scope, rate).await +} + +pub async fn mq_set_subscribe_rate_core( + state: &AppState, + conn_id: &str, + scope: PolicyScope, + rate: SubscribeRate, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Set subscribe rate").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.set_subscribe_rate(&scope, rate).await +} + +pub async fn mq_set_backlog_quota_core( + state: &AppState, + conn_id: &str, + scope: PolicyScope, + quota: BacklogQuota, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Set backlog quota").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.set_backlog_quota(&scope, quota).await +} + +pub async fn mq_set_retention_core( + state: &AppState, + conn_id: &str, + scope: PolicyScope, + retention: RetentionPolicy, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Set retention").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.set_retention(&scope, retention).await +} + +pub async fn mq_get_effective_policies_core( + state: &AppState, + conn_id: &str, + scope: PolicyScope, +) -> Result { + let adapter = get_adapter(state, conn_id).await?; + adapter.get_effective_policies(&scope).await +} + +// ---- Permissions ---- + +pub async fn mq_grant_permission_core( + state: &AppState, + conn_id: &str, + scope: PolicyScope, + role: String, + actions: Vec, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Grant permission").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.grant_permission(&scope, &role, actions).await +} + +pub async fn mq_revoke_permission_core( + state: &AppState, + conn_id: &str, + scope: PolicyScope, + role: String, +) -> Result<(), String> { + ensure_connection_writable(state, conn_id, "Revoke permission").await?; + let adapter = get_adapter(state, conn_id).await?; + adapter.revoke_permission(&scope, &role).await +} + +pub async fn mq_list_permissions_core( + state: &AppState, + conn_id: &str, + scope: PolicyScope, +) -> Result { + let adapter = get_adapter(state, conn_id).await?; + adapter.list_permissions(&scope).await +} + +// ---- Client tokens ---- + +pub async fn mq_issue_token_core( + state: &AppState, + conn_id: &str, + req: MqTokenIssueRequest, +) -> Result { + ensure_connection_writable(state, conn_id, "Issue MQ token").await?; + let cfg = state.configs.read().await.get(conn_id).cloned().ok_or("Connection not found")?; + let mqc = MqAdminConfig::from_connection(&cfg)?; + let signing_config = mqc + .token_signing + .as_ref() + .filter(|config| config.is_configured()) + .ok_or("Token signing is not configured for this MQ connection. Edit the connection and configure Broker Token signing with an HS256 secret or RS256 private key.")?; + + let now = Utc::now(); + let signed = sign_pulsar_token(signing_config, &req, now.timestamp())?; + let expires_at = + signed.expires_at_unix.and_then(|value| Utc.timestamp_opt(value, 0).single()).map(|value| value.to_rfc3339()); + let record = MqTokenRecord { + id: Uuid::new_v4().to_string(), + connection_id: conn_id.to_string(), + subject: req.subject.trim().to_string(), + algorithm: signing_config.algorithm, + token_fingerprint: signed.fingerprint, + scope: req.scope.clone(), + actions: req.actions.clone(), + expires_at, + created_at: now.to_rfc3339(), + note: req.note.as_deref().unwrap_or_default().trim().to_string(), + }; + + state.storage.save_mq_token_record(&record).await?; + Ok(MqIssuedToken { token: signed.token, record }) +} + +pub async fn mq_list_token_records_core( + state: &AppState, + conn_id: &str, + subject: Option, +) -> Result, String> { + if !state.configs.read().await.contains_key(conn_id) { + return Err("Connection not found".to_string()); + } + let subject = subject.as_deref().map(str::trim).filter(|value| !value.is_empty()); + state.storage.load_mq_token_records(conn_id, subject).await +} + +// ---- Monitoring ---- + +pub async fn mq_get_backlog_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, + sub: Option, +) -> Result { + let adapter = get_adapter(state, conn_id).await?; + adapter.get_backlog(&topic, sub.as_deref()).await +} + +// ---- Raw request (escape hatch) ---- + +pub async fn mq_raw_request_core(state: &AppState, conn_id: &str, req: MqRawRequest) -> Result { + if req.is_mutating() { + ensure_connection_writable(state, conn_id, "MQ admin write").await?; + } + let adapter = get_adapter(state, conn_id).await?; + adapter.raw_request(req).await +} + +// --------------------------------------------------------------------------- +// Helper +// --------------------------------------------------------------------------- + +async fn get_adapter( + state: &AppState, + conn_id: &str, +) -> Result, String> { + let cfg = state.configs.read().await.get(conn_id).cloned().ok_or("Connection not found")?; + let mqc = state.mq_admin_config_for_connection(conn_id, &cfg).await?; + state.mq_registry.get_or_build_config(conn_id, mqc).await +} + +async fn ensure_connection_writable(state: &AppState, conn_id: &str, operation: &str) -> Result<(), String> { + let configs = state.configs.read().await; + if let Some(config) = configs.get(conn_id) { + if config.read_only { + return Err(format!( + "Read-only mode: connection '{}' has read-only protection enabled. {} blocked.", + config.name, operation + )); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::models::connection::{ConnectionConfig, DatabaseType}; + use crate::storage::Storage; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn mq_connection(read_only: bool) -> ConnectionConfig { + ConnectionConfig { + id: "readonly-mq".to_string(), + name: "Read only MQ".to_string(), + db_type: DatabaseType::MessageQueue, + driver_profile: Some("pulsar".to_string()), + driver_label: Some("Apache Pulsar".to_string()), + url_params: None, + host: "127.0.0.1".to_string(), + port: 8080, + username: String::new(), + password: String::new(), + database: None, + visible_databases: None, + attached_databases: Vec::new(), + color: None, + transport_layers: Vec::new(), + connect_timeout_secs: 5, + query_timeout_secs: 30, + idle_timeout_secs: 60, + keepalive_interval_secs: crate::models::connection::default_keepalive_interval_secs(), + ssl: false, + ca_cert_path: String::new(), + client_cert_path: String::new(), + client_key_path: String::new(), + sysdba: false, + oracle_connection_type: None, + connection_string: None, + redis_connection_mode: None, + redis_sentinel_master: String::new(), + redis_sentinel_nodes: String::new(), + redis_sentinel_username: String::new(), + redis_sentinel_password: String::new(), + redis_sentinel_tls: false, + redis_cluster_nodes: String::new(), + redis_key_separator: ":".to_string(), + etcd_endpoints: String::new(), + gbase_server: String::new(), + external_config: Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://127.0.0.1:8080", + "auth": { "kind": "none" } + })), + jdbc_driver_class: None, + jdbc_driver_paths: Vec::new(), + one_time: false, + read_only, + } + } + + fn mq_connection_with_token_signing() -> ConnectionConfig { + let mut config = mq_connection(false); + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://127.0.0.1:8080", + "auth": { "kind": "none" }, + "tokenSigning": { + "algorithm": "hs256", + "key": "broker-signing-secret" + } + })); + config + } + + async fn test_state_with(config: ConnectionConfig) -> (AppState, std::path::PathBuf) { + let stamp = + SystemTime::now().duration_since(UNIX_EPOCH).expect("system time should be after UNIX epoch").as_nanos(); + let dir = std::env::temp_dir().join(format!("dbx-mq-service-test-{stamp}")); + std::fs::create_dir_all(&dir).expect("failed to create test directory"); + let storage = Storage::open(&dir.join("storage.db")).await.expect("failed to open test storage"); + let state = AppState::new_with_plugin_dir(storage, dir.join("plugins")); + state.configs.write().await.insert(config.id.clone(), config); + (state, dir) + } + + #[tokio::test] + async fn mutating_service_calls_block_read_only_connections_before_adapter_build() { + let (state, dir) = test_state_with(mq_connection(true)).await; + let err = mq_create_topic_core( + &state, + "readonly-mq", + TopicRef { + tenant: "public".to_string(), + namespace: "default".to_string(), + topic: "orders".to_string(), + persistent: true, + partitioned: None, + }, + None, + ) + .await + .expect_err("read-only write should fail"); + + assert!(err.contains("Read-only mode")); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn mutating_raw_requests_block_read_only_connections_before_adapter_build() { + let (state, dir) = test_state_with(mq_connection(true)).await; + let err = mq_raw_request_core( + &state, + "readonly-mq", + MqRawRequest { + method: "POST".to_string(), + path: "/admin/v2/tenants".to_string(), + query: None, + body: Some(serde_json::json!({ "adminRoles": [] })), + }, + ) + .await + .expect_err("read-only raw write should fail"); + + assert!(err.contains("Read-only mode")); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn peek_messages_rejects_counts_above_service_limit_before_adapter_call() { + let (state, dir) = test_state_with(mq_connection(false)).await; + let err = mq_peek_messages_core( + &state, + "readonly-mq", + TopicRef { + tenant: "public".to_string(), + namespace: "default".to_string(), + topic: "orders".to_string(), + persistent: true, + partitioned: None, + }, + "sub-a".to_string(), + 101, + ) + .await + .expect_err("peek count above the service limit should fail"); + + assert!(err.contains("between 1 and 100")); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn issue_token_saves_record_without_plaintext() { + let (state, dir) = test_state_with(mq_connection_with_token_signing()).await; + + let issued = mq_issue_token_core( + &state, + "readonly-mq", + MqTokenIssueRequest { + subject: "rt-erp-server".to_string(), + expires_in_seconds: Some(3600), + scope: None, + actions: vec![AuthAction::Consume], + note: Some("integration test".to_string()), + }, + ) + .await + .expect("token issuance should succeed"); + + assert!(!issued.token.is_empty()); + assert_eq!(issued.record.subject, "rt-erp-server"); + let records = mq_list_token_records_core(&state, "readonly-mq", Some("rt-erp-server".to_string())) + .await + .expect("listing token records should succeed"); + assert_eq!(records.len(), 1); + assert_eq!(records[0], issued.record); + let record_json = serde_json::to_value(&records[0]).expect("record should serialize to JSON"); + assert!(record_json.get("token").is_none()); + let _ = std::fs::remove_dir_all(dir); + } +} diff --git a/crates/dbx-core/src/mq/token.rs b/crates/dbx-core/src/mq/token.rs new file mode 100644 index 000000000..c0813f0a0 --- /dev/null +++ b/crates/dbx-core/src/mq/token.rs @@ -0,0 +1,121 @@ +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::mq::types::{MqTokenIssueRequest, MqTokenSigningAlgorithm, MqTokenSigningConfig}; + +#[derive(Debug, Clone)] +pub struct SignedPulsarToken { + pub token: String, + pub fingerprint: String, + pub expires_at_unix: Option, +} + +#[derive(Debug, Serialize)] +struct PulsarTokenClaims<'a> { + sub: &'a str, + iat: i64, + #[serde(skip_serializing_if = "Option::is_none")] + exp: Option, +} + +pub fn sign_pulsar_token( + config: &MqTokenSigningConfig, + req: &MqTokenIssueRequest, + now_unix: i64, +) -> Result { + let subject = req.subject.trim(); + if subject.is_empty() { + return Err("Token subject is required".to_string()); + } + + // Validate subject format + if subject.len() > 256 { + return Err("Token subject exceeds maximum length of 256 characters".to_string()); + } + + // Pulsar typically accepts alphanumeric characters plus common separators + // Allow: letters, digits, hyphen, underscore, dot, @ + if !subject.chars().all(|c| c.is_alphanumeric() || matches!(c, '-' | '_' | '.' | '@')) { + return Err(format!( + "Token subject contains invalid characters. Only alphanumeric, '-', '_', '.', and '@' are allowed. Got: {}", + subject + )); + } + + let key = config.key.trim(); + if key.is_empty() { + return Err("Token signing key is required".to_string()); + } + + let expires_at_unix = match req.expires_in_seconds { + Some(seconds) if seconds <= 0 => return Err("Token expiry must be greater than zero seconds".to_string()), + Some(seconds) => Some(now_unix.saturating_add(seconds)), + None => None, + }; + let claims = PulsarTokenClaims { sub: subject, iat: now_unix, exp: expires_at_unix }; + let algorithm = jwt_algorithm(config.algorithm); + let header = Header::new(algorithm); + let encoding_key = encoding_key(config.algorithm, key)?; + let token = encode(&header, &claims, &encoding_key).map_err(|err| format!("Failed to sign Pulsar token: {err}"))?; + let fingerprint = token_fingerprint(&token); + + Ok(SignedPulsarToken { token, fingerprint, expires_at_unix }) +} + +pub fn token_fingerprint(token: &str) -> String { + let digest = Sha256::digest(token.as_bytes()); + format!("sha256:{}", hex_lower(&digest)) +} + +fn jwt_algorithm(algorithm: MqTokenSigningAlgorithm) -> Algorithm { + match algorithm { + MqTokenSigningAlgorithm::Hs256 => Algorithm::HS256, + MqTokenSigningAlgorithm::Rs256 => Algorithm::RS256, + } +} + +fn encoding_key(algorithm: MqTokenSigningAlgorithm, key: &str) -> Result { + match algorithm { + MqTokenSigningAlgorithm::Hs256 => Ok(EncodingKey::from_secret(key.as_bytes())), + MqTokenSigningAlgorithm::Rs256 => { + let normalized = normalize_rsa_key_material(key)?; + EncodingKey::from_rsa_pem(normalized.as_bytes()).map_err(|err| format!("Invalid RS256 private key: {err}")) + } + } +} + +fn normalize_rsa_key_material(key: &str) -> Result { + // If already in PEM format, return as-is + if key.contains("-----BEGIN") { + return Ok(key.to_string()); + } + + // Try to decode as base64 + match BASE64.decode(key.as_bytes()) { + Ok(bytes) => String::from_utf8(bytes).map_err(|e| { + format!( + "RSA key appears to be base64-encoded but decodes to invalid UTF-8: {}. \ + Please provide the key in PEM format (with -----BEGIN/END headers) or as valid base64.", + e + ) + }), + Err(e) => Err(format!( + "RSA key is not in PEM format and failed to decode as base64: {}. \ + Please provide the key in PEM format (with -----BEGIN RSA PRIVATE KEY----- headers) \ + or as valid base64-encoded data.", + e + )), + } +} + +fn hex_lower(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + out +} diff --git a/crates/dbx-core/src/mq/types.rs b/crates/dbx-core/src/mq/types.rs new file mode 100644 index 000000000..792b887d2 --- /dev/null +++ b/crates/dbx-core/src/mq/types.rs @@ -0,0 +1,564 @@ +//! Shared resource, statistics, and configuration types for the message queue +//! admin console. All types are `serde`-serializable in `camelCase` so they map +//! 1:1 to the frontend TypeScript definitions in `apps/desktop/src/types/mq.ts`. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// Which message queue system an admin connection targets. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum MqSystemKind { + Pulsar, + Kafka, + #[serde(rename = "rocketmq")] + RocketMq, +} + +impl MqSystemKind { + pub fn as_str(self) -> &'static str { + match self { + MqSystemKind::Pulsar => "pulsar", + MqSystemKind::Kafka => "kafka", + MqSystemKind::RocketMq => "rocketmq", + } + } +} + +/// Capability flags. The frontend reads these to show/hide functionality, and +/// the adapter computes them from the detected server version so unsupported +/// features are hidden rather than failing at call time. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MqCapabilities { + pub supports_tenants: bool, + pub supports_namespaces: bool, + pub supports_partitioned_topics: bool, + pub supports_subscriptions: bool, + pub supports_create_subscription: bool, + pub supports_reset_cursor: bool, + pub supports_skip_messages: bool, + pub supports_clear_backlog: bool, + pub supports_peek_messages: bool, + pub supports_expire_messages: bool, + pub supports_rate_limits: bool, + pub supports_backlog_quota: bool, + pub supports_retention: bool, + pub supports_permissions: bool, + pub supports_geo_replication: bool, + pub supports_token_management: bool, + pub supports_raw_admin_api: bool, +} + +impl Default for MqCapabilities { + fn default() -> Self { + Self { + supports_tenants: false, + supports_namespaces: false, + supports_partitioned_topics: false, + supports_subscriptions: false, + supports_create_subscription: false, + supports_reset_cursor: false, + supports_skip_messages: false, + supports_clear_backlog: false, + supports_peek_messages: false, + supports_expire_messages: false, + supports_rate_limits: false, + supports_backlog_quota: false, + supports_retention: false, + supports_permissions: false, + supports_geo_replication: false, + supports_token_management: false, + supports_raw_admin_api: false, + } + } +} + +/// Result of a connectivity test, including the detected server version and how +/// it was determined (probe vs. fallback) so the UI can warn appropriately. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MqClusterInfo { + pub system_kind: MqSystemKind, + /// Raw version string reported by the broker (e.g. `3.1.2`), if available. + pub server_version: Option, + /// Version profile the adapter resolved to (e.g. `3.1.x`). + pub resolved_profile: String, + /// How the version was determined: `probed` | `pinned` | `fallback`. + pub version_detection: String, + pub capabilities: MqCapabilities, + /// Optional free-form extra fields (cluster list, broker count, ...). + #[serde(default, skip_serializing_if = "serde_json::Value::is_null")] + pub extra: serde_json::Value, +} + +// --------------------------------------------------------------------------- +// Token signing +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum MqTokenSigningAlgorithm { + Hs256, + Rs256, +} + +impl MqTokenSigningAlgorithm { + pub fn as_str(self) -> &'static str { + match self { + Self::Hs256 => "hs256", + Self::Rs256 => "rs256", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MqTokenSigningConfig { + pub algorithm: MqTokenSigningAlgorithm, + #[serde(default)] + pub key: String, +} + +impl MqTokenSigningConfig { + pub fn is_configured(&self) -> bool { + !self.key.trim().is_empty() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MqTokenIssueRequest { + pub subject: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expires_in_seconds: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scope: Option, + #[serde(default)] + pub actions: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub note: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MqTokenRecord { + pub id: String, + pub connection_id: String, + pub subject: String, + pub algorithm: MqTokenSigningAlgorithm, + pub token_fingerprint: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scope: Option, + #[serde(default)] + pub actions: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expires_at: Option, + pub created_at: String, + #[serde(default)] + pub note: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MqIssuedToken { + pub token: String, + pub record: MqTokenRecord, +} + +// --------------------------------------------------------------------------- +// Tenant +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TenantInfo { + pub name: String, + #[serde(default)] + pub admin_roles: Vec, + #[serde(default)] + pub allowed_clusters: Vec, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TenantConfig { + #[serde(default)] + pub admin_roles: Vec, + #[serde(default)] + pub allowed_clusters: Vec, +} + +// --------------------------------------------------------------------------- +// Namespace +// --------------------------------------------------------------------------- + +/// A fully-qualified namespace reference (`tenant/namespace`). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NamespaceRef { + pub tenant: String, + pub namespace: String, +} + +impl NamespaceRef { + /// `tenant/namespace` + pub fn path(&self) -> String { + format!("{}/{}", self.tenant, self.namespace) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NamespaceInfo { + pub tenant: String, + pub namespace: String, + #[serde(default)] + pub admin_roles: Vec, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NamespaceConfig { + /// Replication clusters to bootstrap the namespace with. Pulsar requires at + /// least one when the cluster runs with geo-replication. + #[serde(default)] + pub clusters: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bundles: Option, +} + +// --------------------------------------------------------------------------- +// Topic +// --------------------------------------------------------------------------- + +/// A fully-qualified topic reference. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TopicRef { + pub tenant: String, + pub namespace: String, + pub topic: String, + /// Whether the topic is persistent (`persistent://`) or not + /// (`non-persistent://`). Defaults to persistent. + #[serde(default = "default_true")] + pub persistent: bool, + /// Optional UI hint used to prefer partitioned stats when the topic came + /// from the partitioned topic list. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub partitioned: Option, +} + +fn default_true() -> bool { + true +} + +impl TopicRef { + /// The URL domain segment: `persistent` or `non-persistent`. + pub fn domain(&self) -> &'static str { + if self.persistent { + "persistent" + } else { + "non-persistent" + } + } + + /// `{tenant}/{namespace}/{topic}` — the path used by most Pulsar endpoints. + pub fn path(&self) -> String { + format!("{}/{}/{}", self.tenant, self.namespace, self.topic) + } + + /// Full topic name, e.g. `persistent://public/default/orders`. + pub fn full_name(&self) -> String { + format!("{}://{}", self.domain(), self.path()) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TopicInfo { + /// Full topic name, e.g. `persistent://public/default/orders`. + pub name: String, + /// Short topic name without the namespace prefix. + pub short_name: String, + pub partitioned: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub partitions: Option, + pub persistent: bool, +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ListTopicsOpts { + /// Include non-persistent topics in the listing. + #[serde(default)] + pub include_non_persistent: bool, +} + +/// Aggregated, UI-friendly topic statistics. Parsed from the version-specific +/// raw stats payload by the adapter's version profile. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TopicStats { + pub msg_rate_in: f64, + pub msg_rate_out: f64, + pub msg_throughput_in: f64, + pub msg_throughput_out: f64, + pub storage_size: i64, + pub backlog_size: i64, + pub msg_in_counter: i64, + pub msg_out_counter: i64, + pub subscription_count: u32, + pub producer_count: u32, + /// Original raw stats JSON, for the detail view / advanced inspection. + #[serde(default, skip_serializing_if = "serde_json::Value::is_null")] + pub raw: serde_json::Value, +} + +// --------------------------------------------------------------------------- +// Subscription / consumers / producers +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SubscriptionInfo { + pub name: String, + #[serde(default)] + pub sub_type: String, + pub msg_backlog: i64, + pub msg_rate_out: f64, + pub msg_throughput_out: f64, + #[serde(default)] + pub consumers: Vec, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConsumerInfo { + pub consumer_name: String, + pub msg_rate_out: f64, + pub msg_throughput_out: f64, + pub available_permits: i64, + #[serde(default)] + pub address: String, + #[serde(default)] + pub client_version: String, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProducerInfo { + pub producer_id: i64, + pub producer_name: String, + pub msg_rate_in: f64, + pub msg_throughput_in: f64, + #[serde(default)] + pub address: String, + #[serde(default)] + pub client_version: String, +} + +/// Where to position a cursor when creating a subscription or resetting it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum ResetPosition { + /// Earliest available message. + Earliest, + /// Latest message (skip the existing backlog). + Latest, + /// A specific point in time (milliseconds since epoch). + Timestamp { timestamp_ms: i64 }, + /// A specific message id. + MessageId { ledger_id: i64, entry_id: i64 }, +} + +/// How many messages to skip on a subscription. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum SkipCount { + All, + Count { count: u32 }, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BacklogStats { + pub msg_backlog: i64, + pub backlog_size: i64, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PeekedMessage { + /// 1-based subscription position passed to the Pulsar Admin API. + pub position: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub publish_time: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub event_time: Option, + #[serde(default)] + pub properties: HashMap, + #[serde(default)] + pub headers: HashMap, + /// Message payload encoded as base64 so binary messages are preserved. + pub payload_base64: String, + /// UTF-8 preview when the payload can be decoded losslessly. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub payload_text: Option, +} + +// --------------------------------------------------------------------------- +// Policy scope (rate limits / quotas / permissions) +// --------------------------------------------------------------------------- + +/// Whether a policy applies at the namespace or topic level. Lets the rate +/// limit / quota / permission methods avoid duplicating namespace vs. topic +/// variants. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "level", rename_all = "camelCase")] +pub enum PolicyScope { + Namespace { tenant: String, namespace: String }, + Topic { tenant: String, namespace: String, topic: String, persistent: bool }, +} + +impl PolicyScope { + pub fn as_namespace_ref(&self) -> Option { + match self { + PolicyScope::Namespace { tenant, namespace } => { + Some(NamespaceRef { tenant: tenant.clone(), namespace: namespace.clone() }) + } + _ => None, + } + } + + pub fn as_topic_ref(&self) -> Option { + match self { + PolicyScope::Topic { tenant, namespace, topic, persistent } => Some(TopicRef { + tenant: tenant.clone(), + namespace: namespace.clone(), + topic: topic.clone(), + persistent: *persistent, + partitioned: None, + }), + _ => None, + } + } + + pub fn is_topic(&self) -> bool { + matches!(self, PolicyScope::Topic { .. }) + } +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PublishRate { + pub publish_throttling_rate_in_msg: i32, + pub publish_throttling_rate_in_byte: i64, +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DispatchRate { + pub dispatch_throttling_rate_in_msg: i32, + pub dispatch_throttling_rate_in_byte: i64, + #[serde(default = "default_rate_period")] + pub rate_period_in_second: i32, +} + +fn default_rate_period() -> i32 { + 1 +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SubscribeRate { + pub subscribe_throttling_rate_per_consumer: i32, + #[serde(default = "default_rate_period")] + pub rate_period_in_second: i32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BacklogQuota { + /// Limit in bytes. `-1` means no size limit. + pub limit_size: i64, + /// Limit in seconds. `-1` means no time limit. + #[serde(default)] + pub limit_time: i32, + /// `producer_request_hold` | `producer_exception` | `consumer_backlog_eviction`. + pub policy: String, + /// `destination_storage` (size) or `message_age` (time). + #[serde(default = "default_backlog_quota_type")] + pub quota_type: String, +} + +fn default_backlog_quota_type() -> String { + "destination_storage".to_string() +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RetentionPolicy { + /// Retention time in minutes. `-1` = infinite. + pub retention_time_in_minutes: i32, + /// Retention size in MB. `-1` = infinite. + pub retention_size_in_mb: i32, +} + +/// Authorization actions that can be granted to a role on a namespace/topic. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AuthAction { + Produce, + Consume, + Functions, + Sources, + Sinks, + #[serde(rename = "packages")] + Packages, +} + +pub type PermissionMap = HashMap>; + +// --------------------------------------------------------------------------- +// Raw request (escape hatch) +// --------------------------------------------------------------------------- + +/// A raw admin REST request, proxied through the adapter to cover any endpoint +/// the typed methods do not. The path is appended to the connection's +/// `admin_url` base — arbitrary hosts are not allowed (SSRF guard). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MqRawRequest { + /// HTTP method: GET / PUT / POST / DELETE. + pub method: String, + /// Path relative to the admin base, e.g. `/admin/v2/tenants`. + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub query: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub body: Option, +} + +impl MqRawRequest { + /// Whether this request mutates server state and therefore must pass the + /// read-only protection check. + pub fn is_mutating(&self) -> bool { + !matches!(self.method.to_ascii_uppercase().as_str(), "GET" | "HEAD" | "OPTIONS") + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MqRawResponse { + pub status: u16, + pub body: serde_json::Value, + /// Set when the response body was not valid JSON; carries the raw text. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text: Option, +} diff --git a/crates/dbx-core/src/mq/util.rs b/crates/dbx-core/src/mq/util.rs new file mode 100644 index 000000000..0620c776e --- /dev/null +++ b/crates/dbx-core/src/mq/util.rs @@ -0,0 +1,39 @@ +//! Shared utilities for the message queue admin module. + +/// Truncate a string to `max` bytes, respecting UTF-8 char boundaries, and +/// append an ellipsis when truncation occurs. +pub(crate) fn truncate(s: &str, max: usize) -> String { + if s.len() <= max { + s.to_string() + } else { + let mut end = max; + while !s.is_char_boundary(end) { + end -= 1; + } + format!("{}…", &s[..end]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn truncate_short_string_unchanged() { + assert_eq!(truncate("hello", 10), "hello"); + } + + #[test] + fn truncate_long_string_adds_ellipsis() { + let result = truncate("abcdefghij", 5); + assert_eq!(result, "abcde…"); + } + + #[test] + fn truncate_handles_multibyte_utf8_boundary() { + let body = format!("a{}", "错".repeat(100)); + let truncated = truncate(&body, 10); + assert!(truncated.ends_with('…')); + assert!(truncated.len() <= 10 + "…".len()); + } +} diff --git a/crates/dbx-core/src/query.rs b/crates/dbx-core/src/query.rs index 820cdf222..c7f636170 100644 --- a/crates/dbx-core/src/query.rs +++ b/crates/dbx-core/src/query.rs @@ -924,6 +924,7 @@ pub async fn do_execute( } PoolKind::Redis(_) => Err("Use Redis-specific commands".to_string()), PoolKind::MongoDb(_) => Err("Use MongoDB-specific commands".to_string()), + PoolKind::MessageQueue => Err("Use Message Queue-specific commands".to_string()), PoolKind::InfluxDb(client) => { let client = client.clone(); let database = pool_key.split(':').nth(1).unwrap_or("default").to_string(); @@ -1585,6 +1586,7 @@ pub async fn execute_statements_in_transaction( | PoolKind::Turso(_) | PoolKind::SqlServer(_) | PoolKind::Agent(_) => TxPath::Explicit, + PoolKind::MessageQueue => TxPath::None, #[cfg(feature = "duckdb-bundled")] PoolKind::DuckDb(_) | PoolKind::Redis(_) diff --git a/crates/dbx-core/src/storage.rs b/crates/dbx-core/src/storage.rs index b23bf48cb..d33d08acf 100644 --- a/crates/dbx-core/src/storage.rs +++ b/crates/dbx-core/src/storage.rs @@ -6,9 +6,13 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::ai::{AiChatMessage, AiConfig, AiConversation}; +use crate::connection_secrets::{ + MQ_AUTH_API_KEY_VALUE_KEY, MQ_AUTH_CLIENT_SECRET_KEY, MQ_AUTH_PASSWORD_KEY, MQ_AUTH_SECRET_PREFIX, + MQ_AUTH_TOKEN_KEY, MQ_TOKEN_SIGNING_KEY, MQ_TOKEN_SIGNING_SECRET_PREFIX, +}; use crate::db::sqlite::{connect_path_create_if_missing, SqliteHandle}; use crate::history::{HistoryEntry, MAX_HISTORY}; -use crate::models::connection::{ConnectionConfig, TransportLayerConfig}; +use crate::models::connection::{ConnectionConfig, DatabaseType, TransportLayerConfig}; use crate::saved_sql::{SavedSqlFile, SavedSqlFolder, SavedSqlLibrary}; const SSH_TUNNEL_SECRET_PREFIX: &str = "ssh_tunnels."; @@ -136,6 +140,22 @@ const SCHEMA_STATEMENTS: &[&str] = &[ byte_size INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL )", + "CREATE TABLE IF NOT EXISTS mq_token_records ( + id TEXT PRIMARY KEY, + connection_id TEXT NOT NULL, + subject TEXT NOT NULL, + algorithm TEXT NOT NULL, + token_fingerprint TEXT NOT NULL, + scope_json TEXT, + actions_json TEXT NOT NULL DEFAULT '[]', + expires_at TEXT, + created_at TEXT NOT NULL, + note TEXT NOT NULL DEFAULT '' + )", + "CREATE INDEX IF NOT EXISTS idx_mq_token_records_connection_subject + ON mq_token_records (connection_id, subject, created_at DESC)", + "CREATE INDEX IF NOT EXISTS idx_mq_token_records_fingerprint + ON mq_token_records (token_fingerprint)", "CREATE TABLE IF NOT EXISTS saved_sql_folders ( id TEXT PRIMARY KEY, connection_id TEXT NOT NULL, @@ -302,6 +322,32 @@ fn scrub_transport_layer_secrets(config: &mut ConnectionConfig) { } } +fn scrub_mq_auth_secrets(config: &mut ConnectionConfig) { + if config.db_type != DatabaseType::MessageQueue { + return; + } + let Some(auth) = mq_auth_object_mut(config.external_config.as_mut()) else { + return; + }; + match mq_auth_kind(auth) { + Some("token") => scrub_json_secret(auth, "token"), + Some("basic") => scrub_json_secret(auth, "password"), + Some(kind) if is_api_key_auth_kind(kind) => scrub_json_secret(auth, "value"), + Some("oauth2") => scrub_json_secret(auth, "clientSecret"), + _ => {} + } +} + +fn scrub_mq_token_signing_secret(config: &mut ConnectionConfig) { + if config.db_type != DatabaseType::MessageQueue { + return; + } + let Some(signing) = mq_token_signing_object_mut(config.external_config.as_mut()) else { + return; + }; + scrub_json_secret(signing, "key"); +} + fn delete_secret_prefix_in_tx( tx: &rusqlite::Transaction<'_>, connection_id: &str, @@ -736,6 +782,8 @@ impl Storage { scrub_transport_layer_secrets(&mut sanitized); sanitized.redis_sentinel_password = String::new(); sanitized.connection_string = None; + scrub_mq_auth_secrets(&mut sanitized); + scrub_mq_token_signing_secret(&mut sanitized); let json = serde_json::to_string(&sanitized).map_err(|e| e.to_string())?; tx.execute("INSERT INTO connections (id, config_json) VALUES (?1, ?2)", params![config_id, json]) @@ -770,6 +818,8 @@ impl Storage { scrub_transport_layer_secrets(&mut sanitized); sanitized.redis_sentinel_password = String::new(); sanitized.connection_string = None; + scrub_mq_auth_secrets(&mut sanitized); + scrub_mq_token_signing_secret(&mut sanitized); let json = serde_json::to_string(&sanitized).map_err(|e| e.to_string())?; tx.execute("INSERT INTO connections (id, config_json) VALUES (?1, ?2)", params![config_id, json]) @@ -817,6 +867,8 @@ impl Storage { ) .map_err(|e| e.to_string())?; } + persist_mq_auth_secrets_in_tx(&tx, &config)?; + persist_mq_token_signing_secret_in_tx(&tx, &config)?; } if configs.is_empty() { @@ -895,10 +947,73 @@ impl Storage { } config.redis_sentinel_password = self.get_secret(&id, "redis_sentinel_password").await?.unwrap_or_default(); config.connection_string = self.get_secret(&id, "connection_string").await?; + let needs_mq_auth_rewrite = self.hydrate_mq_auth_secrets(&id, &mut config).await?; + let needs_mq_token_signing_rewrite = self.hydrate_mq_token_signing_secret(&id, &mut config).await?; + let needs_mq_secret_rewrite = needs_mq_auth_rewrite || needs_mq_token_signing_rewrite; + if needs_mq_secret_rewrite { + let mut sanitized = config.clone().canonicalized(); + scrub_mq_auth_secrets(&mut sanitized); + scrub_mq_token_signing_secret(&mut sanitized); + let sanitized_json = serde_json::to_string(&sanitized).map_err(|e| e.to_string())?; + let update_id = id.clone(); + self.with_conn(move |conn| { + conn.execute( + "UPDATE connections SET config_json = ?1 WHERE id = ?2", + params![sanitized_json, update_id], + ) + .map(|_| ()) + .map_err(|e| e.to_string()) + }) + .await?; + } configs.push(config.canonicalized()); } Ok(configs) } + + async fn hydrate_mq_auth_secrets( + &self, + connection_id: &str, + config: &mut ConnectionConfig, + ) -> Result { + if config.db_type != DatabaseType::MessageQueue { + return Ok(false); + } + let Some(auth) = mq_auth_object_mut(config.external_config.as_mut()) else { + return Ok(false); + }; + + let needs_rewrite = match mq_auth_kind(auth) { + Some("token") => hydrate_mq_json_secret(self, connection_id, MQ_AUTH_TOKEN_KEY, auth, "token").await?, + Some("basic") => { + hydrate_mq_json_secret(self, connection_id, MQ_AUTH_PASSWORD_KEY, auth, "password").await? + } + Some(kind) if is_api_key_auth_kind(kind) => { + hydrate_mq_json_secret(self, connection_id, MQ_AUTH_API_KEY_VALUE_KEY, auth, "value").await? + } + Some("oauth2") => { + hydrate_mq_json_secret(self, connection_id, MQ_AUTH_CLIENT_SECRET_KEY, auth, "clientSecret").await? + } + _ => false, + }; + + Ok(needs_rewrite) + } + + async fn hydrate_mq_token_signing_secret( + &self, + connection_id: &str, + config: &mut ConnectionConfig, + ) -> Result { + if config.db_type != DatabaseType::MessageQueue { + return Ok(false); + } + let Some(signing) = mq_token_signing_object_mut(config.external_config.as_mut()) else { + return Ok(false); + }; + + hydrate_mq_json_secret(self, connection_id, MQ_TOKEN_SIGNING_KEY, signing, "key").await + } } // Saved SQL @@ -1166,6 +1281,76 @@ impl Storage { } } +// MQ token records + +#[cfg(feature = "mq-admin")] +impl Storage { + pub async fn save_mq_token_record(&self, record: &crate::mq::MqTokenRecord) -> Result<(), String> { + let record = record.clone(); + self.with_conn(move |conn| { + let scope_json = record + .scope + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| e.to_string())?; + let actions_json = serde_json::to_string(&record.actions).map_err(|e| e.to_string())?; + conn.execute( + "INSERT OR REPLACE INTO mq_token_records \ + (id, connection_id, subject, algorithm, token_fingerprint, scope_json, actions_json, expires_at, created_at, note) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + record.id, + record.connection_id, + record.subject, + record.algorithm.as_str(), + record.token_fingerprint, + scope_json, + actions_json, + record.expires_at, + record.created_at, + record.note + ], + ) + .map(|_| ()) + .map_err(|e| e.to_string()) + }) + .await + } + + pub async fn load_mq_token_records( + &self, + connection_id: &str, + subject: Option<&str>, + ) -> Result, String> { + let connection_id = connection_id.to_string(); + let subject = subject.map(str::to_string); + self.with_conn(move |conn| { + let sql = if subject.is_some() { + "SELECT id, connection_id, subject, algorithm, token_fingerprint, scope_json, actions_json, expires_at, created_at, note \ + FROM mq_token_records WHERE connection_id = ?1 AND subject = ?2 ORDER BY created_at DESC" + } else { + "SELECT id, connection_id, subject, algorithm, token_fingerprint, scope_json, actions_json, expires_at, created_at, note \ + FROM mq_token_records WHERE connection_id = ?1 ORDER BY created_at DESC" + }; + let mut stmt = conn.prepare(sql).map_err(|e| e.to_string())?; + let rows = if let Some(subject) = subject { + stmt.query_map(params![connection_id, subject], mq_token_record_from_row) + .map_err(|e| e.to_string())? + .collect::, _>>() + .map_err(|e| e.to_string())? + } else { + stmt.query_map(params![connection_id], mq_token_record_from_row) + .map_err(|e| e.to_string())? + .collect::, _>>() + .map_err(|e| e.to_string())? + }; + Ok(rows) + }) + .await + } +} + // Layout impl Storage { @@ -1480,6 +1665,163 @@ fn persist_secret_in_tx( Ok(()) } +fn persist_mq_auth_secrets_in_tx(tx: &rusqlite::Transaction<'_>, config: &ConnectionConfig) -> Result<(), String> { + if config.db_type != DatabaseType::MessageQueue { + delete_secret_prefix_in_tx(tx, &config.id, MQ_AUTH_SECRET_PREFIX)?; + return Ok(()); + } + + let Some(auth) = mq_auth_object(config.external_config.as_ref()) else { + delete_secret_prefix_in_tx(tx, &config.id, MQ_AUTH_SECRET_PREFIX)?; + return Ok(()); + }; + + match mq_auth_kind(auth) { + Some("none") => delete_secret_prefix_in_tx(tx, &config.id, MQ_AUTH_SECRET_PREFIX)?, + Some("token") => replace_mq_auth_secret_in_tx(tx, &config.id, MQ_AUTH_TOKEN_KEY, auth, "token")?, + Some("basic") => replace_mq_auth_secret_in_tx(tx, &config.id, MQ_AUTH_PASSWORD_KEY, auth, "password")?, + Some(kind) if is_api_key_auth_kind(kind) => { + replace_mq_auth_secret_in_tx(tx, &config.id, MQ_AUTH_API_KEY_VALUE_KEY, auth, "value")? + } + Some("oauth2") => { + replace_mq_auth_secret_in_tx(tx, &config.id, MQ_AUTH_CLIENT_SECRET_KEY, auth, "clientSecret")? + } + _ => delete_secret_prefix_in_tx(tx, &config.id, MQ_AUTH_SECRET_PREFIX)?, + } + + Ok(()) +} + +fn replace_mq_auth_secret_in_tx( + tx: &rusqlite::Transaction<'_>, + connection_id: &str, + key: &str, + auth: &serde_json::Map, + field: &str, +) -> Result<(), String> { + let current = auth.get(field).and_then(serde_json::Value::as_str).filter(|secret| !secret.is_empty()); + let existing = if current.is_none() { get_secret_in_tx(tx, connection_id, key)? } else { None }; + delete_secret_prefix_in_tx(tx, connection_id, MQ_AUTH_SECRET_PREFIX)?; + match current { + Some(secret) => persist_secret_in_tx(tx, connection_id, key, secret), + None => match existing { + Some(secret) => persist_secret_in_tx(tx, connection_id, key, &secret), + None => Ok(()), + }, + } +} + +fn get_secret_in_tx(tx: &rusqlite::Transaction<'_>, connection_id: &str, key: &str) -> Result, String> { + tx.query_row( + "SELECT secret FROM connection_secrets WHERE connection_id = ?1 AND key = ?2", + params![connection_id, key], + |row| row.get(0), + ) + .optional() + .map_err(|e| e.to_string()) +} + +fn persist_mq_token_signing_secret_in_tx( + tx: &rusqlite::Transaction<'_>, + config: &ConnectionConfig, +) -> Result<(), String> { + if config.db_type != DatabaseType::MessageQueue { + delete_secret_prefix_in_tx(tx, &config.id, MQ_TOKEN_SIGNING_SECRET_PREFIX)?; + return Ok(()); + } + + let Some(signing) = mq_token_signing_object(config.external_config.as_ref()) else { + delete_secret_prefix_in_tx(tx, &config.id, MQ_TOKEN_SIGNING_SECRET_PREFIX)?; + return Ok(()); + }; + + persist_json_secret_if_present_in_tx(tx, &config.id, MQ_TOKEN_SIGNING_KEY, signing, "key") +} + +fn persist_json_secret_if_present_in_tx( + tx: &rusqlite::Transaction<'_>, + connection_id: &str, + key: &str, + auth: &serde_json::Map, + field: &str, +) -> Result<(), String> { + if let Some(secret) = auth.get(field).and_then(serde_json::Value::as_str).filter(|secret| !secret.is_empty()) { + persist_secret_in_tx(tx, connection_id, key, secret)?; + } + Ok(()) +} + +async fn hydrate_mq_json_secret( + storage: &Storage, + connection_id: &str, + key: &str, + auth: &mut serde_json::Map, + field: &str, +) -> Result { + if let Some(secret) = auth.get(field).and_then(serde_json::Value::as_str).filter(|secret| !secret.is_empty()) { + storage.set_secret(connection_id, key, secret).await?; + Ok(true) + } else if let Some(secret) = storage.get_secret(connection_id, key).await? { + auth.insert(field.to_string(), serde_json::Value::String(secret)); + Ok(false) + } else { + Ok(false) + } +} + +fn scrub_json_secret(auth: &mut serde_json::Map, field: &str) { + if auth.contains_key(field) { + auth.insert(field.to_string(), serde_json::Value::String(String::new())); + } +} + +fn mq_auth_kind(auth: &serde_json::Map) -> Option<&str> { + auth.get("kind").and_then(serde_json::Value::as_str) +} + +fn mq_auth_object(value: Option<&serde_json::Value>) -> Option<&serde_json::Map> { + value?.get("auth")?.as_object() +} + +fn mq_auth_object_mut( + value: Option<&mut serde_json::Value>, +) -> Option<&mut serde_json::Map> { + value?.get_mut("auth")?.as_object_mut() +} + +fn mq_token_signing_object(value: Option<&serde_json::Value>) -> Option<&serde_json::Map> { + value?.get("tokenSigning")?.as_object() +} + +fn mq_token_signing_object_mut( + value: Option<&mut serde_json::Value>, +) -> Option<&mut serde_json::Map> { + value?.get_mut("tokenSigning")?.as_object_mut() +} + +fn is_api_key_auth_kind(kind: &str) -> bool { + matches!(kind, "apiKey" | "api_key" | "apikey") +} + +#[cfg(feature = "mq-admin")] +fn mq_token_record_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let algorithm: String = row.get(3)?; + let scope_json: Option = row.get(5)?; + let actions_json: String = row.get(6)?; + Ok(crate::mq::MqTokenRecord { + id: row.get(0)?, + connection_id: row.get(1)?, + subject: row.get(2)?, + algorithm: serde_json::from_value(serde_json::Value::String(algorithm)).map_err(map_from_sql_err)?, + token_fingerprint: row.get(4)?, + scope: scope_json.as_deref().map(serde_json::from_str).transpose().map_err(map_from_sql_err)?, + actions: serde_json::from_str(&actions_json).map_err(map_from_sql_err)?, + expires_at: row.get(7)?, + created_at: row.get(8)?, + note: row.get(9)?, + }) +} + fn map_from_sql_err(err: serde_json::Error) -> rusqlite::Error { rusqlite::Error::FromSqlConversionFailure(0, rusqlite::types::Type::Text, Box::new(err)) } @@ -1487,6 +1829,8 @@ fn map_from_sql_err(err: serde_json::Error) -> rusqlite::Error { #[cfg(test)] mod tests { use super::{DesktopIconTheme, DesktopSettings, Storage}; + use crate::connection_secrets::{MQ_AUTH_PASSWORD_KEY, MQ_AUTH_TOKEN_KEY, MQ_TOKEN_SIGNING_KEY}; + use crate::models::connection::{ConnectionConfig, DatabaseType}; use std::time::{SystemTime, UNIX_EPOCH}; fn temp_db_path(name: &str) -> std::path::PathBuf { @@ -1494,6 +1838,203 @@ mod tests { std::env::temp_dir().join(format!("dbx-storage-{name}-{}-{stamp}.db", std::process::id())) } + fn mq_connection(id: &str, token: &str) -> ConnectionConfig { + ConnectionConfig { + id: id.to_string(), + name: "Pulsar".to_string(), + db_type: DatabaseType::MessageQueue, + driver_profile: Some("pulsar".to_string()), + driver_label: Some("Apache Pulsar".to_string()), + url_params: None, + host: "127.0.0.1".to_string(), + port: 8080, + username: String::new(), + password: String::new(), + database: None, + visible_databases: None, + attached_databases: Vec::new(), + color: None, + transport_layers: Vec::new(), + connect_timeout_secs: 30, + query_timeout_secs: 300, + idle_timeout_secs: 600, + keepalive_interval_secs: crate::models::connection::default_keepalive_interval_secs(), + ssl: false, + ca_cert_path: String::new(), + client_cert_path: String::new(), + client_key_path: String::new(), + sysdba: false, + oracle_connection_type: None, + connection_string: None, + redis_connection_mode: None, + redis_sentinel_master: String::new(), + redis_sentinel_nodes: String::new(), + redis_sentinel_username: String::new(), + redis_sentinel_password: String::new(), + redis_sentinel_tls: false, + redis_cluster_nodes: String::new(), + redis_key_separator: ":".to_string(), + etcd_endpoints: String::new(), + gbase_server: String::new(), + external_config: Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://127.0.0.1:8080", + "auth": { + "kind": "token", + "token": token + } + })), + jdbc_driver_class: None, + jdbc_driver_paths: Vec::new(), + one_time: false, + read_only: false, + } + } + + async fn raw_connection_json(storage: &Storage, id: &str) -> String { + let id = id.to_string(); + storage + .with_conn(move |conn| { + conn.query_row("SELECT config_json FROM connections WHERE id = ?1", [id], |row| row.get::<_, String>(0)) + .map_err(|e| e.to_string()) + }) + .await + .unwrap() + } + + async fn insert_raw_connection(storage: &Storage, config: &ConnectionConfig) { + let id = config.id.clone(); + let json = serde_json::to_string(config).unwrap(); + storage + .with_conn(move |conn| { + conn.execute("INSERT INTO connections (id, config_json) VALUES (?1, ?2)", rusqlite::params![id, json]) + .map(|_| ()) + .map_err(|e| e.to_string()) + }) + .await + .unwrap(); + } + + fn mq_token(config: &ConnectionConfig) -> Option<&str> { + config.external_config.as_ref()?.get("auth")?.get("token")?.as_str() + } + + fn mq_token_signing_key(config: &ConnectionConfig) -> Option<&str> { + config.external_config.as_ref()?.get("tokenSigning")?.get("key")?.as_str() + } + + #[tokio::test] + async fn save_connections_moves_mq_auth_token_to_secret_table_and_restores_it() { + let path = temp_db_path("mq-token-secrets"); + let storage = Storage::open(&path).await.unwrap(); + + storage.save_connections(&[mq_connection("pulsar", "mq-token-secret")]).await.unwrap(); + + let raw_json = raw_connection_json(&storage, "pulsar").await; + assert!(!raw_json.contains("mq-token-secret")); + let persisted: ConnectionConfig = serde_json::from_str(&raw_json).unwrap(); + assert_eq!(mq_token(&persisted), Some("")); + assert_eq!(storage.get_secret("pulsar", MQ_AUTH_TOKEN_KEY).await.unwrap().as_deref(), Some("mq-token-secret")); + + let loaded = storage.load_connections().await.unwrap(); + assert_eq!(loaded.len(), 1); + assert_eq!(mq_token(&loaded[0]), Some("mq-token-secret")); + } + + #[tokio::test] + async fn metadata_save_scrubs_mq_auth_token_and_preserves_existing_secret() { + let path = temp_db_path("mq-token-metadata"); + let storage = Storage::open(&path).await.unwrap(); + + let original = mq_connection("pulsar", "existing-token"); + storage.save_connections(&[original.clone()]).await.unwrap(); + + let mut metadata = original; + metadata.name = "Pulsar renamed".to_string(); + if let Some(auth) = metadata.external_config.as_mut().and_then(|value| value.get_mut("auth")) { + auth["token"] = serde_json::Value::String("new-token-that-should-not-persist".to_string()); + } + + storage.save_connection_metadata_preserving_secrets(&[metadata]).await.unwrap(); + + let raw_json = raw_connection_json(&storage, "pulsar").await; + assert!(!raw_json.contains("existing-token")); + assert!(!raw_json.contains("new-token-that-should-not-persist")); + assert_eq!(storage.get_secret("pulsar", MQ_AUTH_TOKEN_KEY).await.unwrap().as_deref(), Some("existing-token")); + + let loaded = storage.load_connections().await.unwrap(); + assert_eq!(loaded[0].name, "Pulsar renamed"); + assert_eq!(mq_token(&loaded[0]), Some("existing-token")); + } + + #[tokio::test] + async fn load_connections_migrates_legacy_mq_auth_token_out_of_config_json() { + let path = temp_db_path("mq-token-legacy-migration"); + let storage = Storage::open(&path).await.unwrap(); + insert_raw_connection(&storage, &mq_connection("pulsar", "legacy-token")).await; + + let loaded = storage.load_connections().await.unwrap(); + + assert_eq!(mq_token(&loaded[0]), Some("legacy-token")); + assert_eq!(storage.get_secret("pulsar", MQ_AUTH_TOKEN_KEY).await.unwrap().as_deref(), Some("legacy-token")); + let raw_json = raw_connection_json(&storage, "pulsar").await; + assert!(!raw_json.contains("legacy-token")); + let persisted: ConnectionConfig = serde_json::from_str(&raw_json).unwrap(); + assert_eq!(mq_token(&persisted), Some("")); + } + + #[tokio::test] + async fn save_connections_deletes_stale_mq_auth_secrets_when_kind_changes() { + let path = temp_db_path("mq-auth-kind-change"); + let storage = Storage::open(&path).await.unwrap(); + storage.save_connections(&[mq_connection("pulsar", "old-token")]).await.unwrap(); + let mut config = mq_connection("pulsar", ""); + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://127.0.0.1:8080", + "auth": { + "kind": "basic", + "username": "admin", + "password": "basic-secret" + } + })); + + storage.save_connections(&[config]).await.unwrap(); + + assert_eq!(storage.get_secret("pulsar", MQ_AUTH_TOKEN_KEY).await.unwrap(), None); + assert_eq!(storage.get_secret("pulsar", MQ_AUTH_PASSWORD_KEY).await.unwrap().as_deref(), Some("basic-secret")); + } + + #[tokio::test] + async fn save_connections_moves_mq_token_signing_key_to_secret_table_and_restores_it() { + let path = temp_db_path("mq-token-signing-secret"); + let storage = Storage::open(&path).await.unwrap(); + let mut config = mq_connection("pulsar", ""); + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": "http://127.0.0.1:8080", + "auth": { "kind": "none" }, + "tokenSigning": { + "algorithm": "hs256", + "key": "broker-signing-secret" + } + })); + + storage.save_connections(&[config]).await.unwrap(); + + let raw_json = raw_connection_json(&storage, "pulsar").await; + assert!(!raw_json.contains("broker-signing-secret")); + let persisted: ConnectionConfig = serde_json::from_str(&raw_json).unwrap(); + assert_eq!(mq_token_signing_key(&persisted), Some("")); + assert_eq!( + storage.get_secret("pulsar", MQ_TOKEN_SIGNING_KEY).await.unwrap().as_deref(), + Some("broker-signing-secret") + ); + + let loaded = storage.load_connections().await.unwrap(); + assert_eq!(mq_token_signing_key(&loaded[0]), Some("broker-signing-secret")); + } + #[tokio::test] async fn desktop_settings_default_to_background_enabled() { let path = temp_db_path("desktop-settings-default"); diff --git a/crates/dbx-core/tests/mq_token.rs b/crates/dbx-core/tests/mq_token.rs new file mode 100644 index 000000000..b114fba9d --- /dev/null +++ b/crates/dbx-core/tests/mq_token.rs @@ -0,0 +1,73 @@ +#![cfg(feature = "mq-admin")] + +use std::time::{SystemTime, UNIX_EPOCH}; + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use dbx_core::mq::token::{sign_pulsar_token, token_fingerprint}; +use dbx_core::mq::{MqTokenIssueRequest, MqTokenRecord, MqTokenSigningAlgorithm, MqTokenSigningConfig}; +use dbx_core::storage::Storage; + +#[test] +fn hs256_token_contains_pulsar_subject_and_expiry() { + let config = MqTokenSigningConfig { algorithm: MqTokenSigningAlgorithm::Hs256, key: "broker-secret".to_string() }; + let req = MqTokenIssueRequest { + subject: "order-consumer".to_string(), + expires_in_seconds: Some(3600), + scope: None, + actions: vec![], + note: None, + }; + + let issued = sign_pulsar_token(&config, &req, 1_700_000_000).unwrap(); + let payload = issued.token.split('.').nth(1).unwrap(); + let payload: serde_json::Value = serde_json::from_slice(&URL_SAFE_NO_PAD.decode(payload).unwrap()).unwrap(); + + assert_eq!(payload.get("sub").and_then(serde_json::Value::as_str), Some("order-consumer")); + assert_eq!(payload.get("iat").and_then(serde_json::Value::as_i64), Some(1_700_000_000)); + assert_eq!(payload.get("exp").and_then(serde_json::Value::as_i64), Some(1_700_003_600)); + assert_eq!(issued.fingerprint, token_fingerprint(&issued.token)); +} + +#[test] +fn token_signing_requires_configured_key() { + let config = MqTokenSigningConfig { algorithm: MqTokenSigningAlgorithm::Hs256, key: " ".to_string() }; + let req = MqTokenIssueRequest { + subject: "order-consumer".to_string(), + expires_in_seconds: None, + scope: None, + actions: vec![], + note: None, + }; + + let err = sign_pulsar_token(&config, &req, 1_700_000_000).unwrap_err(); + assert!(err.contains("signing key")); +} + +#[tokio::test] +async fn token_records_are_saved_without_token_plaintext() { + let storage = Storage::open(&temp_db_path("mq-token-records")).await.unwrap(); + let record = MqTokenRecord { + id: "rec-1".to_string(), + connection_id: "conn-1".to_string(), + subject: "order-consumer".to_string(), + algorithm: MqTokenSigningAlgorithm::Hs256, + token_fingerprint: "sha256:abc".to_string(), + scope: None, + actions: vec![], + expires_at: Some("2026-06-14T12:00:00Z".to_string()), + created_at: "2026-06-14T11:00:00Z".to_string(), + note: "issued for test".to_string(), + }; + + storage.save_mq_token_record(&record).await.unwrap(); + let records = storage.load_mq_token_records("conn-1", Some("order-consumer")).await.unwrap(); + + assert_eq!(records, vec![record]); + let json = serde_json::to_value(&records[0]).unwrap(); + assert!(json.get("token").is_none()); +} + +fn temp_db_path(name: &str) -> std::path::PathBuf { + let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + std::env::temp_dir().join(format!("dbx-{name}-{}-{stamp}.db", std::process::id())) +} diff --git a/crates/dbx-web/Cargo.toml b/crates/dbx-web/Cargo.toml index 0f2be4f25..cabb31bbd 100644 --- a/crates/dbx-web/Cargo.toml +++ b/crates/dbx-web/Cargo.toml @@ -8,6 +8,10 @@ license = "Apache-2.0" name = "dbx-web" path = "src/main.rs" +[features] +default = ["mq-admin"] +mq-admin = ["dbx-core/mq-admin"] + [dependencies] dbx-core = { path = "../dbx-core" } redis = { version = "0.32", features = ["tokio-comp"] } diff --git a/crates/dbx-web/src/main.rs b/crates/dbx-web/src/main.rs index d2beb0b52..a9a0b6238 100644 --- a/crates/dbx-web/src/main.rs +++ b/crates/dbx-web/src/main.rs @@ -32,6 +32,56 @@ fn web_body_limit_bytes() -> usize { mb.saturating_mul(1024 * 1024) } +#[cfg(feature = "mq-admin")] +fn add_mq_routes(router: Router>) -> Router> { + router + .route("/mq/test-connection", post(routes::mq::test_connection)) + .route("/mq/tenants/list", post(routes::mq::list_tenants)) + .route("/mq/tenants/get", post(routes::mq::get_tenant)) + .route("/mq/tenants/create", post(routes::mq::create_tenant)) + .route("/mq/tenants/update", post(routes::mq::update_tenant)) + .route("/mq/tenants/delete", post(routes::mq::delete_tenant)) + .route("/mq/namespaces/list", post(routes::mq::list_namespaces)) + .route("/mq/namespaces/create", post(routes::mq::create_namespace)) + .route("/mq/namespaces/delete", post(routes::mq::delete_namespace)) + .route("/mq/namespaces/policies", post(routes::mq::get_namespace_policies)) + .route("/mq/topics/list", post(routes::mq::list_topics)) + .route("/mq/topics/create", post(routes::mq::create_topic)) + .route("/mq/topics/delete", post(routes::mq::delete_topic)) + .route("/mq/topics/update-partitions", post(routes::mq::update_partitions)) + .route("/mq/topics/stats", post(routes::mq::get_topic_stats)) + .route("/mq/topics/internal-stats", post(routes::mq::get_topic_internal_stats)) + .route("/mq/subscriptions/list", post(routes::mq::list_subscriptions)) + .route("/mq/subscriptions/create", post(routes::mq::create_subscription)) + .route("/mq/subscriptions/delete", post(routes::mq::delete_subscription)) + .route("/mq/subscriptions/skip-messages", post(routes::mq::skip_messages)) + .route("/mq/subscriptions/reset-cursor", post(routes::mq::reset_cursor)) + .route("/mq/subscriptions/clear-backlog", post(routes::mq::clear_backlog)) + .route("/mq/subscriptions/peek-messages", post(routes::mq::peek_messages)) + .route("/mq/subscriptions/expire-messages", post(routes::mq::expire_messages)) + .route("/mq/producers/list", post(routes::mq::list_producers)) + .route("/mq/consumers/list", post(routes::mq::list_consumers)) + .route("/mq/topics/unload", post(routes::mq::unload_topic)) + .route("/mq/policies/publish-rate", post(routes::mq::set_publish_rate)) + .route("/mq/policies/dispatch-rate", post(routes::mq::set_dispatch_rate)) + .route("/mq/policies/subscribe-rate", post(routes::mq::set_subscribe_rate)) + .route("/mq/policies/backlog-quota", post(routes::mq::set_backlog_quota)) + .route("/mq/policies/retention", post(routes::mq::set_retention)) + .route("/mq/policies/effective", post(routes::mq::get_effective_policies)) + .route("/mq/permissions/grant", post(routes::mq::grant_permission)) + .route("/mq/permissions/revoke", post(routes::mq::revoke_permission)) + .route("/mq/permissions/list", post(routes::mq::list_permissions)) + .route("/mq/tokens/issue", post(routes::mq::issue_token)) + .route("/mq/tokens/list", post(routes::mq::list_token_records)) + .route("/mq/monitoring/backlog", post(routes::mq::get_backlog)) + .route("/mq/raw", post(routes::mq::raw_request)) +} + +#[cfg(not(feature = "mq-admin"))] +fn add_mq_routes(router: Router>) -> Router> { + router +} + #[tokio::main] async fn main() { tracing_subscriber::fmt() @@ -341,7 +391,9 @@ async fn main() { "/app-settings/pinned-tree-node-ids", get(routes::app_settings::load_pinned_tree_node_ids).post(routes::app_settings::save_pinned_tree_node_ids), ) - .route("/app-settings/config/decrypt", post(routes::app_settings::decrypt_config)) + .route("/app-settings/config/decrypt", post(routes::app_settings::decrypt_config)); + + let api = add_mq_routes(api) .layer(middleware::from_fn_with_state(web_state.clone(), auth::auth_middleware)) .with_state(web_state.clone()); diff --git a/crates/dbx-web/src/routes/connection.rs b/crates/dbx-web/src/routes/connection.rs index 0ea572031..5cf89fe46 100644 --- a/crates/dbx-web/src/routes/connection.rs +++ b/crates/dbx-web/src/routes/connection.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::sync::Arc; use axum::extract::State; @@ -50,6 +51,7 @@ pub async fn test_connection( // Clean up any pool keys created for the temporary connection, including // database-scoped keys like "__test_uuid:database". app.remove_connection_pools(&temp_id).await; + app.reset_connection_transport_for_config(&temp_id, &config).await; app.configs.write().await.remove(&temp_id); match result { @@ -66,6 +68,8 @@ pub async fn connect_db( let app = &state.app; let connection_id = config.id.clone(); + app.remove_connection_pools(&connection_id).await; + app.reset_connection_transport_for_config(&connection_id, &config).await; app.configs.write().await.insert(connection_id.clone(), config.clone()); app.get_or_create_pool(&connection_id, None).await.map_err(AppError)?; @@ -120,26 +124,86 @@ pub async fn save_connections( Json(body): Json, ) -> Result, AppError> { state.app.storage.save_connections(&body.configs).await.map_err(AppError)?; - cache_connection_configs(&state, &body.configs).await; + let sync = sync_connection_configs(&state, &body.configs).await; + remove_connection_pools_for_connection_ids(&state, &sync.connection_pool_ids_to_drop).await; + drop_mq_adapters_for_connection_ids(&state, &sync.mq_adapter_ids_to_drop).await; Ok(Json(())) } pub async fn load_connections(State(state): State>) -> Result>, AppError> { let configs = state.app.storage.load_connections().await.map_err(AppError)?; - cache_connection_configs(&state, &configs).await; + let sync = sync_connection_configs(&state, &configs).await; + remove_connection_pools_for_connection_ids(&state, &sync.connection_pool_ids_to_drop).await; + drop_mq_adapters_for_connection_ids(&state, &sync.mq_adapter_ids_to_drop).await; Ok(Json(configs)) } -async fn cache_connection_configs(state: &WebState, configs: &[ConnectionConfig]) { +struct ConnectionConfigSync { + mq_adapter_ids_to_drop: Vec, + connection_pool_ids_to_drop: Vec, +} + +async fn sync_connection_configs(state: &WebState, configs: &[ConnectionConfig]) -> ConnectionConfigSync { + let saved_ids: HashSet<&str> = configs.iter().map(|config| config.id.as_str()).collect(); + let mut mq_adapter_ids_to_drop = HashSet::new(); + let mut connection_pool_ids_to_drop = HashSet::new(); let mut runtime_configs = state.app.configs.write().await; + runtime_configs.retain(|id, existing| { + if saved_ids.contains(id.as_str()) || is_transient_runtime_config_id(id) { + true + } else { + connection_pool_ids_to_drop.insert(id.clone()); + if existing.db_type == dbx_core::models::connection::DatabaseType::MessageQueue { + mq_adapter_ids_to_drop.insert(id.clone()); + } + false + } + }); for config in configs { - runtime_configs.insert(config.id.clone(), config.clone()); + if config.db_type == dbx_core::models::connection::DatabaseType::MessageQueue { + mq_adapter_ids_to_drop.insert(config.id.clone()); + } + if let Some(previous) = runtime_configs.insert(config.id.clone(), config.clone()) { + if previous.db_type == dbx_core::models::connection::DatabaseType::MessageQueue { + mq_adapter_ids_to_drop.insert(config.id.clone()); + } + if &previous != config { + connection_pool_ids_to_drop.insert(config.id.clone()); + } + } + } + ConnectionConfigSync { + mq_adapter_ids_to_drop: mq_adapter_ids_to_drop.into_iter().collect(), + connection_pool_ids_to_drop: connection_pool_ids_to_drop.into_iter().collect(), + } +} + +fn is_transient_runtime_config_id(id: &str) -> bool { + id.starts_with("__test_") || id.starts_with("__visible_draft_") +} + +#[cfg(feature = "mq-admin")] +async fn drop_mq_adapters_for_connection_ids(state: &WebState, connection_ids: &[String]) { + for connection_id in connection_ids { + state.app.mq_registry.drop_connection(connection_id).await; + } +} + +#[cfg(not(feature = "mq-admin"))] +async fn drop_mq_adapters_for_connection_ids(_state: &WebState, _connection_ids: &[String]) {} + +async fn remove_connection_pools_for_connection_ids(state: &WebState, connection_ids: &[String]) { + for connection_id in connection_ids { + state.app.remove_connection_pools(connection_id).await; } } #[cfg(test)] mod tests { - use super::{disconnect_db, save_connections, DisconnectRequest, SaveConnectionsRequest}; + use super::{ + connect_db, disconnect_db, load_connections, save_connections, ConnectRequest, DisconnectRequest, + SaveConnectionsRequest, + }; use crate::state::{LoginRateLimit, WebState}; use axum::extract::State; use axum::Json; @@ -148,6 +212,8 @@ mod tests { use dbx_core::storage::Storage; use std::collections::{HashMap, HashSet}; use std::sync::Arc; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; use tokio::sync::{Mutex, RwLock}; fn sqlite_config(id: &str, path: &str) -> ConnectionConfig { @@ -196,6 +262,19 @@ mod tests { } } + fn mq_config(id: &str, admin_url: &str) -> ConnectionConfig { + let mut config = sqlite_config(id, ""); + config.name = "Pulsar".to_string(); + config.db_type = DatabaseType::MessageQueue; + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": admin_url, + "auth": { "kind": "none" }, + "pinnedVersion": "3.1" + })); + config + } + async fn test_web_state() -> (Arc, std::path::PathBuf) { let dir = std::env::temp_dir().join(format!("dbx-web-test-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&dir).unwrap(); @@ -215,6 +294,34 @@ mod tests { (state, dir) } + async fn spawn_pulsar_clusters_server() -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + break; + }; + tokio::spawn(async move { + let mut buf = [0_u8; 1024]; + let Ok(n) = stream.read(&mut buf).await else { + return; + }; + let request = String::from_utf8_lossy(&buf[..n]); + let status = + if request.starts_with("GET /admin/v2/clusters ") { "200 OK" } else { "404 Not Found" }; + let body = if status.starts_with("200") { r#"["ec-pulsar"]"# } else { r#"{"reason":"missing"}"# }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()).await; + }); + } + }); + format!("http://{addr}") + } + #[tokio::test] async fn save_connections_updates_runtime_config_cache() { let (state, dir) = test_web_state().await; @@ -232,6 +339,133 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + #[tokio::test] + async fn save_connections_drops_cached_mq_adapter_for_updated_config() { + let (state, dir) = test_web_state().await; + let initial = mq_config("mq-conn", "http://127.0.0.1:8080"); + state.app.configs.write().await.insert(initial.id.clone(), initial.clone()); + state.app.connections.write().await.insert(initial.id.clone(), PoolKind::MessageQueue); + let first = state.app.mq_registry.get_or_build(&initial).await.unwrap(); + + let updated = mq_config("mq-conn", "http://127.0.0.1:8081"); + let result = + save_connections(State(state.clone()), Json(SaveConnectionsRequest { configs: vec![updated.clone()] })) + .await; + assert!(result.is_ok()); + + let cached_admin_url = state + .app + .configs + .read() + .await + .get("mq-conn") + .and_then(|config| config.external_config.as_ref()) + .and_then(|external| external.get("adminUrl")) + .and_then(serde_json::Value::as_str) + .map(str::to_string); + assert_eq!(cached_admin_url.as_deref(), Some("http://127.0.0.1:8081")); + + let second = state.app.mq_registry.get_or_build(&updated).await.unwrap(); + assert!(!Arc::ptr_eq(&first, &second)); + assert!(!state.app.connections.read().await.contains_key(&initial.id)); + + let _ = std::fs::remove_dir_all(dir); + } + + #[cfg(feature = "mq-admin")] + #[tokio::test] + async fn connect_db_rebuilds_mq_adapter_for_updated_config_with_same_id() { + let (state, dir) = test_web_state().await; + let initial = mq_config("mq-conn", "http://127.0.0.1:8080"); + state.app.configs.write().await.insert(initial.id.clone(), initial.clone()); + state.app.connections.write().await.insert(initial.id.clone(), PoolKind::MessageQueue); + let first = state.app.mq_registry.get_or_build(&initial).await.unwrap(); + + let updated = mq_config("mq-conn", &spawn_pulsar_clusters_server().await); + let result = connect_db(State(state.clone()), Json(ConnectRequest { config: updated.clone() })).await; + assert!(result.is_ok()); + + let second = state.app.mq_registry.get_or_build(&updated).await.unwrap(); + assert!(!Arc::ptr_eq(&first, &second)); + + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn load_connections_drops_cached_pool_for_updated_config() { + let (state, dir) = test_web_state().await; + let initial = mq_config("mq-conn", "http://127.0.0.1:8080"); + let updated = mq_config("mq-conn", "http://127.0.0.1:8081"); + state.app.storage.save_connections(&[updated.clone()]).await.unwrap(); + state.app.configs.write().await.insert(initial.id.clone(), initial.clone()); + state.app.connections.write().await.insert(initial.id.clone(), PoolKind::MessageQueue); + + let result = load_connections(State(state.clone())).await; + assert!(result.is_ok()); + + let configs = state.app.configs.read().await; + let cached_admin_url = configs + .get("mq-conn") + .and_then(|config| config.external_config.as_ref()) + .and_then(|external| external.get("adminUrl")) + .and_then(serde_json::Value::as_str); + assert_eq!(cached_admin_url, Some("http://127.0.0.1:8081")); + drop(configs); + assert!(!state.app.connections.read().await.contains_key(&initial.id)); + + let _ = std::fs::remove_dir_all(dir); + } + + #[cfg(feature = "mq-admin")] + #[tokio::test] + async fn save_connections_removes_deleted_runtime_config_and_mq_adapter() { + let (state, dir) = test_web_state().await; + let kept = sqlite_config("kept", &dir.join("kept.db").to_string_lossy()); + let removed = mq_config("removed-mq", "http://127.0.0.1:8080"); + { + let mut configs = state.app.configs.write().await; + configs.insert(kept.id.clone(), kept.clone()); + configs.insert(removed.id.clone(), removed.clone()); + } + let stale = state.app.mq_registry.get_or_build(&removed).await.unwrap(); + + let result = + save_connections(State(state.clone()), Json(SaveConnectionsRequest { configs: vec![kept.clone()] })).await; + assert!(result.is_ok()); + + let configs = state.app.configs.read().await; + assert!(configs.contains_key("kept")); + assert!(!configs.contains_key("removed-mq")); + drop(configs); + + let rebuilt = state.app.mq_registry.get_or_build(&removed).await.unwrap(); + assert!(!Arc::ptr_eq(&stale, &rebuilt)); + + let _ = std::fs::remove_dir_all(dir); + } + + #[cfg(feature = "mq-admin")] + #[tokio::test] + async fn save_connections_removes_deleted_connection_pools() { + let (state, dir) = test_web_state().await; + let kept = sqlite_config("kept", &dir.join("kept.db").to_string_lossy()); + let removed = mq_config("removed-mq", "http://127.0.0.1:8080"); + { + let mut configs = state.app.configs.write().await; + configs.insert(kept.id.clone(), kept.clone()); + configs.insert(removed.id.clone(), removed.clone()); + } + state.app.connections.write().await.insert(removed.id.clone(), PoolKind::MessageQueue); + + let result = + save_connections(State(state.clone()), Json(SaveConnectionsRequest { configs: vec![kept.clone()] })).await; + assert!(result.is_ok()); + + assert!(!state.app.connections.read().await.contains_key(&removed.id)); + + let _ = std::fs::remove_dir_all(dir); + } + #[tokio::test] async fn disconnect_db_keeps_connections_with_similar_prefixes() { let (state, dir) = test_web_state().await; @@ -285,6 +519,26 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + #[cfg(feature = "mq-admin")] + #[tokio::test] + async fn disconnect_db_drops_cached_mq_adapter() { + let (state, dir) = test_web_state().await; + let config = mq_config("mq-conn", "http://127.0.0.1:8080"); + state.app.configs.write().await.insert(config.id.clone(), config.clone()); + state.app.connections.write().await.insert(config.id.clone(), PoolKind::MessageQueue); + let first = state.app.mq_registry.get_or_build(&config).await.unwrap(); + + let result = + disconnect_db(State(state.clone()), Json(DisconnectRequest { connection_id: config.id.clone() })).await; + assert!(result.is_ok()); + + assert!(!state.app.connections.read().await.contains_key(&config.id)); + let second = state.app.mq_registry.get_or_build(&config).await.unwrap(); + assert!(!Arc::ptr_eq(&first, &second)); + + let _ = std::fs::remove_dir_all(dir); + } + #[tokio::test] async fn disconnect_db_removes_visible_database_draft_config() { let (state, dir) = test_web_state().await; diff --git a/crates/dbx-web/src/routes/mod.rs b/crates/dbx-web/src/routes/mod.rs index 68de749c3..06dc593a3 100644 --- a/crates/dbx-web/src/routes/mod.rs +++ b/crates/dbx-web/src/routes/mod.rs @@ -9,6 +9,8 @@ pub mod history; pub mod jdbc; pub mod layout; pub mod mongo; +#[cfg(feature = "mq-admin")] +pub mod mq; pub mod plugins; pub mod query; pub mod redis; diff --git a/crates/dbx-web/src/routes/mq.rs b/crates/dbx-web/src/routes/mq.rs new file mode 100644 index 000000000..675c8b31d --- /dev/null +++ b/crates/dbx-web/src/routes/mq.rs @@ -0,0 +1,711 @@ +//! Web (Axum) routes for message queue admin operations. +//! +//! Mirrors the desktop command layer, sharing the same `dbx_core::mq::service::*_core` +//! functions, with read-only protection for mutating operations. + +use axum::extract::State; +use axum::Json; +use std::sync::Arc; + +use crate::error::AppError; +use crate::state::WebState; + +// Request wrappers for endpoints that need more than just connection_id. + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ConnReq { + connection_id: String, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct TenantReq { + connection_id: String, + name: String, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CreateTenantReq { + connection_id: String, + name: String, + config: dbx_core::mq::TenantConfig, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct UpdateTenantReq { + connection_id: String, + name: String, + config: dbx_core::mq::TenantConfig, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DeleteTenantReq { + connection_id: String, + name: String, + #[serde(default)] + force: bool, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ListNamespacesReq { + connection_id: String, + tenant: String, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CreateNamespaceReq { + connection_id: String, + ns: dbx_core::mq::NamespaceRef, + config: dbx_core::mq::NamespaceConfig, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DeleteNamespaceReq { + connection_id: String, + ns: dbx_core::mq::NamespaceRef, + #[serde(default)] + force: bool, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct NamespacePoliciesReq { + connection_id: String, + ns: dbx_core::mq::NamespaceRef, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ListTopicsReq { + connection_id: String, + ns: dbx_core::mq::NamespaceRef, + opts: dbx_core::mq::ListTopicsOpts, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CreateTopicReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + partitions: Option, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DeleteTopicReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + #[serde(default)] + force: bool, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct UpdatePartitionsReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + partitions: u32, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct TopicReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CreateSubscriptionReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + pos: dbx_core::mq::ResetPosition, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DeleteSubscriptionReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + #[serde(default)] + force: bool, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SkipMessagesReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + count: dbx_core::mq::SkipCount, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ResetCursorReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + pos: dbx_core::mq::ResetPosition, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SubscriptionReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PeekMessagesReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + count: u32, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ExpireMessagesReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + expire_seconds: i64, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SetPublishRateReq { + connection_id: String, + scope: dbx_core::mq::PolicyScope, + rate: dbx_core::mq::PublishRate, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SetDispatchRateReq { + connection_id: String, + scope: dbx_core::mq::PolicyScope, + rate: dbx_core::mq::DispatchRate, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SetSubscribeRateReq { + connection_id: String, + scope: dbx_core::mq::PolicyScope, + rate: dbx_core::mq::SubscribeRate, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SetBacklogQuotaReq { + connection_id: String, + scope: dbx_core::mq::PolicyScope, + quota: dbx_core::mq::BacklogQuota, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SetRetentionReq { + connection_id: String, + scope: dbx_core::mq::PolicyScope, + retention: dbx_core::mq::RetentionPolicy, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PolicyScopeReq { + connection_id: String, + scope: dbx_core::mq::PolicyScope, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct GrantPermissionReq { + connection_id: String, + scope: dbx_core::mq::PolicyScope, + role: String, + actions: Vec, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RevokePermissionReq { + connection_id: String, + scope: dbx_core::mq::PolicyScope, + role: String, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct IssueTokenReq { + connection_id: String, + req: dbx_core::mq::MqTokenIssueRequest, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ListTokenRecordsReq { + connection_id: String, + subject: Option, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct BacklogReq { + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: Option, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RawRequestReq { + connection_id: String, + req: dbx_core::mq::MqRawRequest, +} + +// ---- Helper: writable check ---- + +async fn ensure_writable(app: &dbx_core::connection::AppState, conn_id: &str, action: &str) -> Result<(), AppError> { + if let Some(name) = dbx_core::query::connection_readonly_name(app, conn_id).await { + return Err(AppError(format!("Read-only connection '{name}'. {action} is blocked."))); + } + Ok(()) +} + +// ---- Handlers ---- + +pub async fn test_connection( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + let result = + dbx_core::mq::service::mq_test_connection_core(&state.app, &req.connection_id).await.map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn list_tenants( + State(state): State>, + Json(req): Json, +) -> Result>, AppError> { + let result = dbx_core::mq::service::mq_list_tenants_core(&state.app, &req.connection_id).await.map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn get_tenant( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + let result = + dbx_core::mq::service::mq_get_tenant_core(&state.app, &req.connection_id, &req.name).await.map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn create_tenant( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Create tenant").await?; + dbx_core::mq::service::mq_create_tenant_core(&state.app, &req.connection_id, &req.name, req.config) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn update_tenant( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Update tenant").await?; + dbx_core::mq::service::mq_update_tenant_core(&state.app, &req.connection_id, &req.name, req.config) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn delete_tenant( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Delete tenant").await?; + dbx_core::mq::service::mq_delete_tenant_core(&state.app, &req.connection_id, &req.name, req.force) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn list_namespaces( + State(state): State>, + Json(req): Json, +) -> Result>, AppError> { + let result = dbx_core::mq::service::mq_list_namespaces_core(&state.app, &req.connection_id, &req.tenant) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn create_namespace( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Create namespace").await?; + dbx_core::mq::service::mq_create_namespace_core(&state.app, &req.connection_id, req.ns, req.config) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn delete_namespace( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Delete namespace").await?; + dbx_core::mq::service::mq_delete_namespace_core(&state.app, &req.connection_id, req.ns, req.force) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn get_namespace_policies( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + let result = dbx_core::mq::service::mq_get_namespace_policies_core(&state.app, &req.connection_id, req.ns) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn list_topics( + State(state): State>, + Json(req): Json, +) -> Result>, AppError> { + let result = dbx_core::mq::service::mq_list_topics_core(&state.app, &req.connection_id, req.ns, req.opts) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn create_topic( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Create topic").await?; + dbx_core::mq::service::mq_create_topic_core(&state.app, &req.connection_id, req.topic, req.partitions) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn delete_topic( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Delete topic").await?; + dbx_core::mq::service::mq_delete_topic_core(&state.app, &req.connection_id, req.topic, req.force) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn update_partitions( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Update partitions").await?; + dbx_core::mq::service::mq_update_partitions_core(&state.app, &req.connection_id, req.topic, req.partitions) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn get_topic_stats( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + let result = dbx_core::mq::service::mq_get_topic_stats_core(&state.app, &req.connection_id, req.topic) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn get_topic_internal_stats( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + let result = dbx_core::mq::service::mq_get_topic_internal_stats_core(&state.app, &req.connection_id, req.topic) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn list_subscriptions( + State(state): State>, + Json(req): Json, +) -> Result>, AppError> { + let result = dbx_core::mq::service::mq_list_subscriptions_core(&state.app, &req.connection_id, req.topic) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn create_subscription( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Create subscription").await?; + dbx_core::mq::service::mq_create_subscription_core(&state.app, &req.connection_id, req.topic, req.sub, req.pos) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn delete_subscription( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Delete subscription").await?; + dbx_core::mq::service::mq_delete_subscription_core(&state.app, &req.connection_id, req.topic, req.sub, req.force) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn skip_messages( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Skip messages").await?; + dbx_core::mq::service::mq_skip_messages_core(&state.app, &req.connection_id, req.topic, req.sub, req.count) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn reset_cursor( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Reset cursor").await?; + dbx_core::mq::service::mq_reset_cursor_core(&state.app, &req.connection_id, req.topic, req.sub, req.pos) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn clear_backlog( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Clear backlog").await?; + dbx_core::mq::service::mq_clear_backlog_core(&state.app, &req.connection_id, req.topic, req.sub) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn peek_messages( + State(state): State>, + Json(req): Json, +) -> Result>, AppError> { + let result = + dbx_core::mq::service::mq_peek_messages_core(&state.app, &req.connection_id, req.topic, req.sub, req.count) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn expire_messages( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Expire messages").await?; + dbx_core::mq::service::mq_expire_messages_core( + &state.app, + &req.connection_id, + req.topic, + req.sub, + req.expire_seconds, + ) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn list_producers( + State(state): State>, + Json(req): Json, +) -> Result>, AppError> { + let result = dbx_core::mq::service::mq_list_producers_core(&state.app, &req.connection_id, req.topic) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn list_consumers( + State(state): State>, + Json(req): Json, +) -> Result>, AppError> { + let result = dbx_core::mq::service::mq_list_consumers_core(&state.app, &req.connection_id, req.topic, req.sub) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn unload_topic(State(state): State>, Json(req): Json) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Unload topic").await?; + dbx_core::mq::service::mq_unload_topic_core(&state.app, &req.connection_id, req.topic).await.map_err(AppError)?; + Ok(Json(())) +} + +pub async fn set_publish_rate( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Set publish rate").await?; + dbx_core::mq::service::mq_set_publish_rate_core(&state.app, &req.connection_id, req.scope, req.rate) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn set_dispatch_rate( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Set dispatch rate").await?; + dbx_core::mq::service::mq_set_dispatch_rate_core(&state.app, &req.connection_id, req.scope, req.rate) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn set_subscribe_rate( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Set subscribe rate").await?; + dbx_core::mq::service::mq_set_subscribe_rate_core(&state.app, &req.connection_id, req.scope, req.rate) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn set_backlog_quota( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Set backlog quota").await?; + dbx_core::mq::service::mq_set_backlog_quota_core(&state.app, &req.connection_id, req.scope, req.quota) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn set_retention( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Set retention").await?; + dbx_core::mq::service::mq_set_retention_core(&state.app, &req.connection_id, req.scope, req.retention) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn get_effective_policies( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + let result = dbx_core::mq::service::mq_get_effective_policies_core(&state.app, &req.connection_id, req.scope) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn grant_permission( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Grant permission").await?; + dbx_core::mq::service::mq_grant_permission_core(&state.app, &req.connection_id, req.scope, req.role, req.actions) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn revoke_permission( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Revoke permission").await?; + dbx_core::mq::service::mq_revoke_permission_core(&state.app, &req.connection_id, req.scope, req.role) + .await + .map_err(AppError)?; + Ok(Json(())) +} + +pub async fn list_permissions( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + let result = dbx_core::mq::service::mq_list_permissions_core(&state.app, &req.connection_id, req.scope) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn issue_token( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + ensure_writable(&state.app, &req.connection_id, "Issue MQ token").await?; + let result = + dbx_core::mq::service::mq_issue_token_core(&state.app, &req.connection_id, req.req).await.map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn list_token_records( + State(state): State>, + Json(req): Json, +) -> Result>, AppError> { + let result = dbx_core::mq::service::mq_list_token_records_core(&state.app, &req.connection_id, req.subject) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn get_backlog( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + let result = dbx_core::mq::service::mq_get_backlog_core(&state.app, &req.connection_id, req.topic, req.sub) + .await + .map_err(AppError)?; + Ok(Json(result)) +} + +pub async fn raw_request( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + if req.req.is_mutating() { + ensure_writable(&state.app, &req.connection_id, "MQ admin write").await?; + } + let result = + dbx_core::mq::service::mq_raw_request_core(&state.app, &req.connection_id, req.req).await.map_err(AppError)?; + Ok(Json(result)) +} diff --git a/docs/mq-quick-start.md b/docs/mq-quick-start.md new file mode 100644 index 000000000..3d4897c3f --- /dev/null +++ b/docs/mq-quick-start.md @@ -0,0 +1,443 @@ +# 消息队列管理功能 - 快速开始 + +## 概述 + +该功能为 DBX 增加了消息队列(Message Queue)管理能力,首批支持 **Apache Pulsar** 的管理操作。架构保留 Kafka、RocketMQ 扩展点,但在适配器完成前 UI 只允许创建 Pulsar 连接。 + +## 功能特性 + +### 已实现(Pulsar 首期核心) +- ✅ **租户管理**:创建、删除、查询、更新租户信息 +- ✅ **命名空间管理**:创建、删除、查询命名空间,配置策略 +- ✅ **主题管理**:创建普通/分区主题,删除、查询统计信息 +- ✅ **订阅管理**:创建、删除订阅,重置游标,清理积压,跳过消息 +- ✅ **生产者/消费者监控**:查看活跃的生产者和消费者 +- ✅ **策略配置**:发布速率、分发速率、订阅速率、积压配额、保留策略 +- ✅ **权限管理**:授予/撤销角色权限 +- ✅ **监控统计**:消息速率、吞吐量、积压大小 +- ✅ **Raw Admin API**:直接发送自定义管理请求(逃生通道) + +### 前端 UI(Pulsar 首期核心可用) +- ✅ 连接配置、控制台入口、租户/命名空间/主题/订阅/监控/生产者消费者/策略/权限/Raw API 面板 +- ⚠️ 后续扩展:完整 i18n、Kafka/RocketMQ 适配器、高级 typed policy(最大生产者/消费者数、TTL、deduplication) + +## 架构设计 + +``` +┌─────────────────────────────────────────────────────────────┐ +│ 前端 (Vue) │ +│ ┌─────────────┐ ┌──────────────┐ ┌──────────────────┐ │ +│ │ MqAdminConsole │ │ TenantsPanel │ │ TopicsPanel │ │ +│ └─────────────┘ └──────────────┘ └──────────────────┘ │ +│ ▼ mq-api.ts (统一 API 层) │ +│ ┌──────────┴───────────┐ │ +│ │ mq-tauri.ts │ mq-http.ts │ │ +└──────┴─────────────┴────────────────────────────────────────┘ + ▼ Tauri Invoke ▼ HTTP Fetch +┌─────────────────────────────────────────────────────────────┐ +│ Rust 后端 (Tauri / Axum) │ +│ ┌────────────────┐ ┌──────────────────┐ │ +│ │ commands/mq_cmd │ ◀─▶ │ routes/mq │ │ +│ └────────────────┘ └──────────────────┘ │ +│ ▼ ▼ │ +│ ┌────────────────────────────────────────────────────┐ │ +│ │ dbx-core/mq/service.rs (*_core) │ │ +│ └────────────────────────────────────────────────────┘ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────┐ │ +│ │ MqAdminRegistry (缓存 adapter 实例) │ │ +│ └────────────────────────────────────────────────────┘ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────┐ │ +│ │ MessageQueueAdmin trait (port.rs) │ │ +│ └────────────────────────────────────────────────────┘ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────┐ │ +│ │ PulsarAdapter (adapters/pulsar.rs) │ │ +│ │ • 版本探测 (pulsar_version.rs) │ │ +│ │ • OAuth2 token 缓存 (auth.rs) │ │ +│ │ • SSRF 防护 │ │ +│ └────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ▼ HTTPS +┌─────────────────────────────────────────────────────────────┐ +│ Apache Pulsar Admin REST API │ +└─────────────────────────────────────────────────────────────┘ +``` + +## 快速配置 + +### 1. 创建 Pulsar 连接 + +在 DBX 中添加新连接,配置如下: + +**基本信息**: +- **数据库类型**:选择 `mq` (Message Queue) +- **连接名称**:例如 "Pulsar Production" +- **Host/Port**:填写任意值(MQ 连接不使用这些字段) + +**扩展配置** (external_config JSON): +```json +{ + "systemKind": "pulsar", + "adminUrl": "https://pulsar.example.com:8443", + "auth": { + "kind": "token", + "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9..." + }, + "tlsSkipVerify": false +} +``` + +### 2. 认证方式 + +#### Token 认证(最常用) +```json +{ + "kind": "token", + "token": "your-jwt-token" +} +``` + +#### Basic 认证 +```json +{ + "kind": "basic", + "username": "admin", + "password": "admin-secret" +} +``` + +#### OAuth2 认证(自动缓存 token) +```json +{ + "kind": "oauth2", + "issuerUrl": "https://auth.example.com/oauth/token", + "clientId": "pulsar-admin", + "clientSecret": "client-secret", + "audience": "https://pulsar.example.com", + "scope": "pulsar:admin" +} +``` + +#### API Key 认证 +```json +{ + "kind": "apiKey", + "header": "X-API-Key", + "value": "your-api-key" +} +``` + +#### 无认证 +```json +{ + "kind": "none" +} +``` + +### 3. 高级选项 + +```json +{ + "systemKind": "pulsar", + "adminUrl": "https://pulsar.example.com:8443", + "auth": { "kind": "token", "token": "..." }, + "tlsSkipVerify": true, // 跳过 TLS 证书验证(开发环境) + "pinnedVersion": "3.1.0" // 强制使用特定 API profile +} +``` + +## 使用示例 + +### 租户管理 + +```typescript +import { mqListTenants, mqCreateTenant } from '@/lib/mq-api' + +// 列出所有租户 +const tenants = await mqListTenants(connectionId) + +// 创建租户 +await mqCreateTenant(connectionId, 'my-tenant', { + adminRoles: ['admin', 'ops'], + allowedClusters: ['prod-cluster'] +}) +``` + +### 命名空间管理 + +```typescript +import { mqListNamespaces, mqCreateNamespace } from '@/lib/mq-api' + +// 列出租户下的命名空间 +const namespaces = await mqListNamespaces(connectionId, 'my-tenant') + +// 创建命名空间 +await mqCreateNamespace(connectionId, { + tenant: 'my-tenant', + namespace: 'my-namespace' +}, {}) +``` + +### 主题管理 + +```typescript +import { mqListTopics, mqCreateTopic, mqGetTopicStats } from '@/lib/mq-api' + +// 列出主题 +const topics = await mqListTopics(connectionId, { + tenant: 'my-tenant', + namespace: 'my-namespace' +}, { + includeNonPersistent: false +}) + +// 创建分区主题 +await mqCreateTopic(connectionId, { + tenant: 'my-tenant', + namespace: 'my-namespace', + topic: 'my-topic', + persistent: true +}, 4) // 4 个分区 + +// 查询主题统计 +const stats = await mqGetTopicStats(connectionId, { + tenant: 'my-tenant', + namespace: 'my-namespace', + topic: 'my-topic', + persistent: true +}) +console.log(`消息速率: ${stats.msgRateIn}/s`) +console.log(`积压大小: ${stats.backlogSize} bytes`) +``` + +### 订阅管理 + +```typescript +import { + mqListSubscriptions, + mqCreateSubscription, + mqResetCursor, + mqClearBacklog +} from '@/lib/mq-api' + +const topicRef = { + tenant: 'my-tenant', + namespace: 'my-namespace', + topic: 'my-topic', + persistent: true +} + +// 列出订阅 +const subs = await mqListSubscriptions(connectionId, topicRef) + +// 创建订阅(从最早消息开始) +await mqCreateSubscription(connectionId, topicRef, 'my-subscription', { + kind: 'earliest' +}) + +// 重置游标到最新 +await mqResetCursor(connectionId, topicRef, 'my-subscription', { + kind: 'latest' +}) + +// 清空积压 +await mqClearBacklog(connectionId, topicRef, 'my-subscription') +``` + +### 策略配置 + +```typescript +import { mqSetPublishRate, mqSetRetention } from '@/lib/mq-api' + +const scope = { + level: 'namespace' as const, + tenant: 'my-tenant', + namespace: 'my-namespace' +} + +// 设置发布速率限制 +await mqSetPublishRate(connectionId, scope, { + publishThrottlingRateInMsg: 1000, // 1000 msg/s + publishThrottlingRateInByte: 1048576 // 1 MB/s +}) + +// 设置保留策略 +await mqSetRetention(connectionId, scope, { + retentionTimeInMinutes: 7 * 24 * 60, // 7 天 + retentionSizeInMb: 10240 // 10 GB +}) +``` + +### 权限管理 + +```typescript +import { mqGrantPermission, mqListPermissions } from '@/lib/mq-api' + +const scope = { + level: 'namespace' as const, + tenant: 'my-tenant', + namespace: 'my-namespace' +} + +// 授予权限 +await mqGrantPermission(connectionId, scope, 'my-app-role', ['produce', 'consume']) + +// 查询权限 +const permissions = await mqListPermissions(connectionId, scope) +console.log(permissions) // { 'my-app-role': ['produce', 'consume'] } +``` + +## 桌面端命令注册 + +所有后端功能已通过 Tauri commands 暴露给前端: + +```rust +// src-tauri/src/lib.rs +.invoke_handler(tauri::generate_handler![ + // ... 其他命令 + commands::mq_cmd::mq_test_connection, + commands::mq_cmd::mq_list_tenants, + commands::mq_cmd::mq_create_tenant, + // ... 共 48 个 mq 命令 +]) +``` + +## Web 端路由注册 + +所有后端功能已通过 Axum routes 暴露为 REST API: + +```rust +// crates/dbx-web/src/main.rs +.route("/mq/test-connection", post(routes::mq::test_connection)) +.route("/mq/tenants/list", post(routes::mq::list_tenants)) +.route("/mq/tenants/create", post(routes::mq::create_tenant)) +// ... 共 48 个路由 +``` + +## 安全特性 + +### 1. 只读保护 +所有变更操作会检查连接的 `read_only` 标志: +```rust +ensure_connection_writable(&state, &connection_id, "Create tenant").await?; +``` + +### 2. SSRF 防护 +Raw request 路径必须是相对当前 Admin base 的绝对路径:必须以 `/` 开头,不能包含 scheme/host,也不能包含 `..` 路径段: +```rust +if path.contains("://") || path.starts_with("//") || !path.starts_with('/') || path.split('/').any(|seg| seg == "..") { + return Err("Raw request path is not safe".to_string()); +} +``` + +### 3. OAuth2 Token 缓存 +避免每次请求都交换 token(60 秒缓存,5 秒提前刷新): +```rust +pub struct OAuth2TokenCache { + token: RwLock>, + config: OAuth2Config, +} +``` + +### 4. 版本兼容性 +自动探测 Pulsar 版本,优雅降级不支持的 API: +```rust +async fn detect_pulsar_version(client: &reqwest::Client, base_url: &str) -> PulsarProfile { + // 探测 /admin/v2/brokers/version + // 返回 3.1.x baseline profile +} +``` + +## 编译与运行 + +### 启用 MQ 功能 +```bash +# 默认已启用(default = ["duckdb-bundled", "mq-admin"]) +cargo build --release + +# 或显式指定 +cargo build --release --features mq-admin +``` + +### 禁用 MQ 功能 +```bash +cargo build --release --no-default-features --features duckdb-bundled +``` + +### 检查编译 +```bash +cargo check --package dbx-core --features mq-admin +cargo check --package dbx --features mq-admin +cargo check --package dbx-web --features mq-admin +``` + +## 后续扩展 + +### 添加新的 MQ 系统(如 Kafka) + +当前版本仅在架构和类型层预留 Kafka/RocketMQ;前端连接对话框不会提供这些选项。新增适配器并完成验证后,再开放 UI 选择。 + +1. 实现 `MessageQueueAdmin` trait: +```rust +// crates/dbx-core/src/mq/adapters/kafka.rs +pub struct KafkaAdapter { + config: Arc, + client: KafkaAdminClient, +} + +#[async_trait] +impl MessageQueueAdmin for KafkaAdapter { + async fn test_connection(&self) -> Result { + // Kafka 特定的连接测试 + } + + async fn list_topics(&self, ns: &NamespaceRef, opts: ListTopicsOpts) -> Result, String> { + // Kafka 使用不同的 topic 模型,适配到统一接口 + } + + // ... 实现其他方法 +} +``` + +2. 在 `MqAdminRegistry::build_adapter` 中注册: +```rust +match mq_config.system_kind { + MqSystemKind::Pulsar => { /* ... */ } + MqSystemKind::Kafka => { + Ok(Arc::new(KafkaAdapter::new(config.clone()).await?)) + } + // ... +} +``` + +3. 适配器、能力位和端到端验证完成后,再在连接对话框开放对应系统选择。 + +## 常见问题 + +### Q: 连接测试失败 "SSL certificate problem" +A: 开发环境可设置 `tlsSkipVerify: true`,生产环境应配置正确的 CA 证书。 + +### Q: 权限不足 "Not authorized" +A: 检查 token 是否有 `admin` 或相应的角色权限。 + +### Q: 某些 API 返回 404 +A: 可能是 Pulsar 版本不支持该 API,检查 `test_connection` 返回的 `capabilities`。 + +### Q: OAuth2 token 一直刷新失败 +A: 检查 `issuerUrl`、`clientId`、`clientSecret`、`audience` 是否正确。 + +### Q: 如何调试 Admin API +A: 使用 Raw API 面板或直接调用 `mqRawRequest`: +```typescript +const response = await mqRawRequest(connectionId, { + method: 'GET', + path: '/admin/v2/clusters', + query: {}, + body: null +}) +console.log(response.body) +``` + + \ No newline at end of file diff --git a/packages/app-tests/openTabsPersistence.test.ts b/packages/app-tests/openTabsPersistence.test.ts index d6ee30e74..33fedc738 100644 --- a/packages/app-tests/openTabsPersistence.test.ts +++ b/packages/app-tests/openTabsPersistence.test.ts @@ -97,6 +97,18 @@ test("serializes table tabs with reload context", () => { ); }); +test("serializes MQ tabs with selected tenant context", () => { + const saved = serializeOpenTabs([ + queryTab({ + mode: "mq", + database: "", + mqTenant: "public", + }), + ]); + + assert.equal(saved[0]?.mqTenant, "public"); +}); + test("serializes evicted result cache handles", () => { const saved = serializeOpenTabs([ queryTab({ @@ -302,6 +314,20 @@ test("restores data and structure tabs with table state", () => { assert.equal(restored.tabs[1]?.structureTableName, "users"); }); +test("restores MQ tabs with selected tenant context", () => { + const raw = JSON.stringify([ + queryTab({ + mode: "mq", + database: "", + mqTenant: "public", + }), + ]); + + const restored = restoreOpenTabsState(raw, "tab-1"); + + assert.equal(restored.tabs[0]?.mqTenant, "public"); +}); + test("query-only restore keeps legacy query tabs without a mode", () => { const raw = JSON.stringify([ { diff --git a/packages/app-tests/sidebarActiveTabTarget.test.ts b/packages/app-tests/sidebarActiveTabTarget.test.ts index 96affee88..7ab4518ba 100644 --- a/packages/app-tests/sidebarActiveTabTarget.test.ts +++ b/packages/app-tests/sidebarActiveTabTarget.test.ts @@ -59,6 +59,33 @@ test("mongo tabs target the matching visible collection node", () => { assert.equal(findSidebarNodeForActiveTab(tab, [flat(collection)])?.id, "events-node"); }); +test("MQ tabs with a selected tenant target the matching tenant node", () => { + const tab: QueryTab = { + id: "tab-1", + title: "Apache Pulsar Admin", + connectionId: "conn-1", + database: "", + sql: "", + isExecuting: false, + mode: "mq", + mqTenant: "public", + }; + const tenant: TreeNode = { + id: "tenant-node", + label: "public", + type: "mq-tenant", + connectionId: "conn-1", + mqTenant: "public", + }; + + assert.deepEqual(activeTabSidebarTarget(tab), { + type: "mq-tenant", + connectionId: "conn-1", + tenant: "public", + }); + assert.equal(findSidebarNodeForActiveTab(tab, [flat(tenant)])?.id, "tenant-node"); +}); + test("saved SQL tabs target the matching visible saved SQL file node", () => { const tab: QueryTab = { id: "tab-1", diff --git a/packages/mcp-server/src/index.ts b/packages/mcp-server/src/index.ts index bfff43280..282d4dba9 100644 --- a/packages/mcp-server/src/index.ts +++ b/packages/mcp-server/src/index.ts @@ -46,7 +46,7 @@ function formatQueryToolResult(result: QueryResult, title?: string) { } export const DBX_CONNECTION_TYPE_DESCRIPTION = - "Database type: postgres, mysql, sqlite, rqlite, redis, duckdb, clickhouse, sqlserver, mongodb, oracle, elasticsearch, etcd, doris, starrocks, manticoresearch, redshift, dameng, kingbase, highgo, vastbase, goldendb, databend, gaussdb, kwdb, yashandb, databricks, saphana, teradata, vertica, firebird, exasol, opengauss, oceanbase-oracle, questdb, gbase, h2, snowflake, trino, hive, db2, informix, influxdb, iris, neo4j, cassandra, bigquery, kylin, sundb, tdengine, iotdb, xugu, jdbc, access"; + "Database type: postgres, mysql, sqlite, rqlite, redis, duckdb, clickhouse, sqlserver, mongodb, oracle, elasticsearch, etcd, doris, starrocks, manticoresearch, redshift, dameng, kingbase, highgo, vastbase, goldendb, databend, gaussdb, kwdb, yashandb, databricks, saphana, teradata, vertica, firebird, exasol, opengauss, oceanbase-oracle, questdb, gbase, h2, snowflake, trino, hive, db2, informix, influxdb, iris, neo4j, cassandra, bigquery, kylin, sundb, tdengine, iotdb, xugu, jdbc, access, mq"; const FILE_CAPABLE_CONNECTION_TYPES = new Set(["sqlite", "duckdb", "access", "h2"]); export function createDbxMcpServer(backend: Backend, options: { isWebMode?: boolean } = {}): McpServer { diff --git a/scripts/check-mq-integration.mjs b/scripts/check-mq-integration.mjs new file mode 100644 index 000000000..0d3d26cbb --- /dev/null +++ b/scripts/check-mq-integration.mjs @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; + +const root = path.resolve(import.meta.dirname, ".."); +const read = (file) => fs.readFileSync(path.join(root, file), "utf8"); + +function assertIncludes(file, needle, message) { + assert.ok(read(file).includes(needle), `${message}\nMissing ${needle} in ${file}`); +} + +function assertMatches(file, pattern, message) { + assert.match(read(file), pattern, `${message}\nPattern ${pattern} not found in ${file}`); +} + +assertMatches("src-tauri/Cargo.toml", /mq-admin\s*=\s*\[[^\]]*"dbx-core\/mq-admin"[^\]]*\]/, "Tauri crate must expose an mq-admin feature that enables dbx-core/mq-admin."); +assertMatches("src-tauri/Cargo.toml", /default\s*=\s*\[[^\]]*"mq-admin"[^\]]*\]/, "Tauri default features must include mq-admin so desktop commands are registered."); +assertMatches("crates/dbx-web/Cargo.toml", /\[features\][\s\S]*mq-admin\s*=\s*\[[^\]]*"dbx-core\/mq-admin"[^\]]*\]/, "Web crate must expose an mq-admin feature that enables dbx-core/mq-admin."); +assertMatches("crates/dbx-web/Cargo.toml", /\[features\][\s\S]*default\s*=\s*\[[^\]]*"mq-admin"[^\]]*\]/, "Web default features must include mq-admin so web routes are registered."); + +assertIncludes("apps/desktop/src/components/layout/ContentArea.vue", "MqAdminConsole", "Main content area must import and render the MQ admin console."); +assertIncludes("apps/desktop/src/components/layout/ContentArea.vue", "activeTab.mode === 'mq'", "Main content area must have an mq mode render branch."); +assertIncludes("apps/desktop/src/components/sidebar/TreeItem.vue", '"mq"', "Sidebar connection handling must be aware of MQ connections."); +assertIncludes("apps/desktop/src/stores/queryStore.ts", "openMqAdmin", "Query store must expose an MQ admin tab opener."); + +const manifest = JSON.parse(read("crates/dbx-core/assets/database-drivers.manifest.json")); +const drivers = Array.isArray(manifest) ? manifest : manifest.drivers; +assert.ok(drivers?.some((driver) => driver.dbType === "mq"), "Driver manifest must include dbType=mq."); + +const mqHttp = read("apps/desktop/src/lib/mq-http.ts"); +assert.ok(!mqHttp.includes('post("/mq/'), "MQ HTTP client must not call unprefixed /mq paths."); +assert.ok(mqHttp.includes('post("/api/mq/'), "MQ HTTP client must call /api/mq paths in web mode."); + +assertIncludes("apps/desktop/src/lib/api.ts", 'mqTestConnection = forward("mqTestConnection")', "MQ frontend calls must use the shared forward() API layer."); +assertIncludes("apps/desktop/src/components/connection/ConnectionDialog.vue", "mqAdminUrl", "Connection dialog must include MQ admin URL fields."); +assertIncludes("apps/desktop/src/components/connection/ConnectionDialog.vue", "external_config", "Connection dialog must submit MQ external_config."); + +const mqConsole = read("apps/desktop/src/components/mq/MqAdminConsole.vue"); +for (const tab of ["policies", "permissions", "raw"]) { + assert.ok(mqConsole.includes(`'${tab}'`), `MQ console must expose a ${tab} tab.`); +} +for (const panel of ["PoliciesPanel.vue", "PermissionsPanel.vue", "RawApiPanel.vue"]) { + assert.ok(fs.existsSync(path.join(root, "apps/desktop/src/components/mq", panel)), `Missing MQ panel: ${panel}`); + assertIncludes(`apps/desktop/src/components/mq/${panel}`, "readOnly", `${panel} must honor read-only mode.`); +} +for (const panel of ["TenantsPanel.vue", "NamespacesPanel.vue", "TopicsPanel.vue", "SubscriptionsPanel.vue"]) { + assertIncludes(`apps/desktop/src/components/mq/${panel}`, "readOnly", `${panel} must disable mutating actions in read-only mode.`); +} + +console.log("MQ integration checks passed"); diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 31e6cd153..82d7e876c 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -13,8 +13,9 @@ name = "dbx_lib" crate-type = ["staticlib", "cdylib", "rlib"] [features] -default = ["duckdb-bundled"] +default = ["duckdb-bundled", "mq-admin"] duckdb-bundled = ["duckdb", "dbx-core/duckdb-bundled"] +mq-admin = ["dbx-core/mq-admin"] [build-dependencies] tauri-build = { version = "2.5.6", features = [] } diff --git a/src-tauri/src/commands/connection.rs b/src-tauri/src/commands/connection.rs index ca5ac8952..13e326ff8 100644 --- a/src-tauri/src/commands/connection.rs +++ b/src-tauri/src/commands/connection.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::sync::Arc; use tauri::State; @@ -167,8 +168,10 @@ async fn connect_agent_pool( #[cfg(test)] mod tests { - use super::mongo_legacy_connect_params; + use super::{load_connection_configs, mongo_legacy_connect_params, save_connection_configs}; + use dbx_core::connection::{AppState, PoolKind}; use dbx_core::models::connection::{ConnectionConfig, DatabaseType}; + use dbx_core::storage::Storage; fn mongodb_config() -> ConnectionConfig { ConnectionConfig { @@ -218,6 +221,29 @@ mod tests { } } + fn mq_config(id: &str, admin_url: &str) -> ConnectionConfig { + let mut config = mongodb_config(); + config.id = id.to_string(); + config.name = "Pulsar".to_string(); + config.db_type = DatabaseType::MessageQueue; + config.driver_profile = None; + config.driver_label = None; + config.url_params = None; + config.host = String::new(); + config.port = 0; + config.username = String::new(); + config.password = String::new(); + config.database = None; + config.connection_string = None; + config.external_config = Some(serde_json::json!({ + "systemKind": "pulsar", + "adminUrl": admin_url, + "auth": { "kind": "none" }, + "pinnedVersion": "3.1" + })); + config + } + #[test] fn mongo_legacy_connect_params_preserve_auth_options() { let config = mongodb_config(); @@ -231,21 +257,210 @@ mod tests { "mongodb://mongouser:secret@172.22.4.42:27017/RestCloud_V45PUB_Gateway?authSource=admin" ); } + + #[cfg(feature = "mq-admin")] + #[tokio::test] + async fn save_connection_configs_updates_runtime_cache_and_drops_mq_adapter() { + let dir = std::env::temp_dir().join(format!("dbx-tauri-conn-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new_with_plugin_dir(storage, dir.join("plugins")); + let initial = mq_config("mq-conn", "http://127.0.0.1:8080"); + state.configs.write().await.insert(initial.id.clone(), initial.clone()); + state.connections.write().await.insert(initial.id.clone(), PoolKind::MessageQueue); + let first = state.mq_registry.get_or_build(&initial).await.unwrap(); + + let updated = mq_config("mq-conn", "http://127.0.0.1:8081"); + save_connection_configs(&state, &[updated.clone()]).await.unwrap(); + + let cached_admin_url = state + .configs + .read() + .await + .get("mq-conn") + .and_then(|config| config.external_config.as_ref()) + .and_then(|external| external.get("adminUrl")) + .and_then(serde_json::Value::as_str) + .map(str::to_string); + assert_eq!(cached_admin_url.as_deref(), Some("http://127.0.0.1:8081")); + + let second = state.mq_registry.get_or_build(&updated).await.unwrap(); + assert!(!std::sync::Arc::ptr_eq(&first, &second)); + assert!(!state.connections.read().await.contains_key(&initial.id)); + + let _ = std::fs::remove_dir_all(dir); + } + + #[cfg(feature = "mq-admin")] + #[tokio::test] + async fn load_connection_configs_syncs_runtime_cache_and_drops_stale_pool() { + let dir = std::env::temp_dir().join(format!("dbx-tauri-conn-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new_with_plugin_dir(storage, dir.join("plugins")); + let initial = mq_config("mq-conn", "http://127.0.0.1:8080"); + let updated = mq_config("mq-conn", "http://127.0.0.1:8081"); + state.storage.save_connections(&[updated.clone()]).await.unwrap(); + state.configs.write().await.insert(initial.id.clone(), initial.clone()); + state.connections.write().await.insert(initial.id.clone(), PoolKind::MessageQueue); + + let loaded = load_connection_configs(&state).await.unwrap(); + + assert_eq!(loaded.len(), 1); + let cached_admin_url = state + .configs + .read() + .await + .get("mq-conn") + .and_then(|config| config.external_config.as_ref()) + .and_then(|external| external.get("adminUrl")) + .and_then(serde_json::Value::as_str) + .map(str::to_string); + assert_eq!(cached_admin_url.as_deref(), Some("http://127.0.0.1:8081")); + assert!(!state.connections.read().await.contains_key(&initial.id)); + + let _ = std::fs::remove_dir_all(dir); + } + + #[cfg(feature = "mq-admin")] + #[tokio::test] + async fn save_connection_configs_removes_deleted_runtime_config_and_mq_adapter() { + let dir = std::env::temp_dir().join(format!("dbx-tauri-conn-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new_with_plugin_dir(storage, dir.join("plugins")); + let kept = mongodb_config(); + let removed = mq_config("removed-mq", "http://127.0.0.1:8080"); + { + let mut configs = state.configs.write().await; + configs.insert(kept.id.clone(), kept.clone()); + configs.insert(removed.id.clone(), removed.clone()); + } + let stale = state.mq_registry.get_or_build(&removed).await.unwrap(); + + save_connection_configs(&state, &[kept.clone()]).await.unwrap(); + + let configs = state.configs.read().await; + assert!(configs.contains_key(&kept.id)); + assert!(!configs.contains_key("removed-mq")); + drop(configs); + + let rebuilt = state.mq_registry.get_or_build(&removed).await.unwrap(); + assert!(!std::sync::Arc::ptr_eq(&stale, &rebuilt)); + + let _ = std::fs::remove_dir_all(dir); + } + + #[cfg(feature = "mq-admin")] + #[tokio::test] + async fn save_connection_configs_removes_deleted_connection_pools() { + let dir = std::env::temp_dir().join(format!("dbx-tauri-conn-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new_with_plugin_dir(storage, dir.join("plugins")); + let kept = mongodb_config(); + let removed = mq_config("removed-mq", "http://127.0.0.1:8080"); + { + let mut configs = state.configs.write().await; + configs.insert(kept.id.clone(), kept.clone()); + configs.insert(removed.id.clone(), removed.clone()); + } + state.connections.write().await.insert(removed.id.clone(), PoolKind::MessageQueue); + + save_connection_configs(&state, &[kept.clone()]).await.unwrap(); + + assert!(!state.connections.read().await.contains_key(&removed.id)); + + let _ = std::fs::remove_dir_all(dir); + } } #[tauri::command] pub async fn save_connections(state: State<'_, Arc>, configs: Vec) -> Result<(), String> { let configs: Vec = configs.into_iter().map(|config| config.canonicalized()).collect(); - state.storage.save_connections(&configs).await + save_connection_configs(state.inner(), &configs).await +} + +async fn save_connection_configs(state: &AppState, configs: &[ConnectionConfig]) -> Result<(), String> { + state.storage.save_connections(configs).await?; + let sync = sync_connection_configs(state, configs).await; + remove_connection_pools_for_connection_ids(state, &sync.connection_pool_ids_to_drop).await; + drop_mq_adapters_for_connection_ids(state, &sync.mq_adapter_ids_to_drop).await; + Ok(()) +} + +struct ConnectionConfigSync { + mq_adapter_ids_to_drop: Vec, + connection_pool_ids_to_drop: Vec, +} + +async fn sync_connection_configs(state: &AppState, configs: &[ConnectionConfig]) -> ConnectionConfigSync { + let saved_ids: HashSet<&str> = configs.iter().map(|config| config.id.as_str()).collect(); + let mut mq_adapter_ids_to_drop = HashSet::new(); + let mut connection_pool_ids_to_drop = HashSet::new(); + let mut runtime_configs = state.configs.write().await; + runtime_configs.retain(|id, existing| { + if saved_ids.contains(id.as_str()) || is_transient_runtime_config_id(id) { + true + } else { + connection_pool_ids_to_drop.insert(id.clone()); + if existing.db_type == DatabaseType::MessageQueue { + mq_adapter_ids_to_drop.insert(id.clone()); + } + false + } + }); + for config in configs { + if config.db_type == DatabaseType::MessageQueue { + mq_adapter_ids_to_drop.insert(config.id.clone()); + } + if let Some(previous) = runtime_configs.insert(config.id.clone(), config.clone()) { + if previous.db_type == DatabaseType::MessageQueue { + mq_adapter_ids_to_drop.insert(config.id.clone()); + } + if &previous != config { + connection_pool_ids_to_drop.insert(config.id.clone()); + } + } + } + ConnectionConfigSync { + mq_adapter_ids_to_drop: mq_adapter_ids_to_drop.into_iter().collect(), + connection_pool_ids_to_drop: connection_pool_ids_to_drop.into_iter().collect(), + } +} + +fn is_transient_runtime_config_id(id: &str) -> bool { + id.starts_with("__test_") || id.starts_with("__visible_draft_") +} + +#[cfg(feature = "mq-admin")] +async fn drop_mq_adapters_for_connection_ids(state: &AppState, connection_ids: &[String]) { + for connection_id in connection_ids { + state.mq_registry.drop_connection(connection_id).await; + } +} + +#[cfg(not(feature = "mq-admin"))] +async fn drop_mq_adapters_for_connection_ids(_state: &AppState, _connection_ids: &[String]) {} + +async fn remove_connection_pools_for_connection_ids(state: &AppState, connection_ids: &[String]) { + for connection_id in connection_ids { + state.remove_connection_pools(connection_id).await; + } } #[tauri::command] pub async fn load_connections(state: State<'_, Arc>) -> Result, String> { - state - .storage - .load_connections() - .await - .map(|configs| configs.into_iter().map(|config| config.canonicalized()).collect()) + load_connection_configs(state.inner()).await +} + +async fn load_connection_configs(state: &AppState) -> Result, String> { + let configs: Vec = + state.storage.load_connections().await?.into_iter().map(|config| config.canonicalized()).collect(); + let sync = sync_connection_configs(state, &configs).await; + remove_connection_pools_for_connection_ids(state, &sync.connection_pool_ids_to_drop).await; + drop_mq_adapters_for_connection_ids(state, &sync.mq_adapter_ids_to_drop).await; + Ok(configs) } #[tauri::command] @@ -481,6 +696,18 @@ pub async fn test_connection(state: State<'_, Arc>, config: Connection .await .map(|_| "Connection successful".to_string()) } + #[cfg(feature = "mq-admin")] + DatabaseType::MessageQueue => { + let mqc = state.mq_admin_config_for_connection(connection_id, &config).await?; + let adapter = state.mq_registry.build_transient_config(mqc).await?; + adapter.test_connection().await?; + Ok("Connection successful".to_string()) + } + #[cfg(not(feature = "mq-admin"))] + DatabaseType::MessageQueue => { + Err("Message queue admin support is not compiled in this build. Rebuild with the 'mq-admin' feature." + .to_string()) + } db_type if database_capabilities::is_agent_type(&db_type) => { test_agent_connection(state.inner(), &config, &host, port).await } @@ -707,6 +934,20 @@ pub async fn connect_db(state: State<'_, Arc>, config: ConnectionConfi db::influxdb_driver::test_connection(&client, connect_timeout).await?; PoolKind::InfluxDb(client) } + #[cfg(feature = "mq-admin")] + DatabaseType::MessageQueue => { + let mqc = state.mq_admin_config_for_connection(&id, &config).await?; + let adapter = state.mq_registry.build_transient_config(mqc).await?; + adapter.test_connection().await?; + PoolKind::MessageQueue + } + #[cfg(not(feature = "mq-admin"))] + DatabaseType::MessageQueue => { + return Err( + "Message queue admin support is not compiled in this build. Rebuild with the 'mq-admin' feature." + .to_string(), + ); + } db_type if database_capabilities::is_agent_type(&db_type) => { connect_agent_pool(state.inner(), &db_config, &host, port).await? } @@ -741,6 +982,7 @@ pub async fn connection_final_proxy_port( #[tauri::command] pub async fn disconnect_db(state: State<'_, Arc>, connection_id: String) -> Result<(), String> { state.remove_connection_pools_detached(&connection_id).await; + drop_mq_adapters_for_connection_ids(state.inner(), std::slice::from_ref(&connection_id)).await; state.reset_connection_transport(&connection_id).await; if connection_id.starts_with("__visible_draft_") { state.configs.write().await.remove(&connection_id); diff --git a/src-tauri/src/commands/mcp.rs b/src-tauri/src/commands/mcp.rs index fe9fe4129..0e1cfc5b8 100644 --- a/src-tauri/src/commands/mcp.rs +++ b/src-tauri/src/commands/mcp.rs @@ -1,3 +1,4 @@ +#[cfg(not(windows))] use std::env; use std::path::Path; use std::process::Command; @@ -276,6 +277,7 @@ fn default_user_shell() -> String { } } +#[cfg(not(windows))] fn shell_command_script(command: &str, args: &[&str]) -> String { let mut words = Vec::with_capacity(args.len() + 1); words.push(shell_quote(command)); @@ -283,6 +285,7 @@ fn shell_command_script(command: &str, args: &[&str]) -> String { format!("printf '%s\\n' {}; {}", shell_quote(SHELL_COMMAND_MARKER), words.join(" ")) } +#[cfg(not(windows))] fn shell_quote(value: &str) -> String { if value.is_empty() { return "''".to_string(); @@ -299,10 +302,13 @@ fn stdout_after_shell_marker(stdout: &str) -> String { #[cfg(test)] mod tests { - use super::{ - bash_login_script, shell_command_script, shell_quote, stdout_after_shell_marker, SHELL_COMMAND_MARKER, - }; + #[cfg(not(windows))] + use super::bash_login_script; + #[cfg(not(windows))] + use super::{shell_command_script, shell_quote}; + use super::{stdout_after_shell_marker, SHELL_COMMAND_MARKER}; + #[cfg(not(windows))] #[test] fn shell_quote_handles_empty_and_single_quotes() { assert_eq!(shell_quote(""), "''"); @@ -310,6 +316,7 @@ mod tests { assert_eq!(shell_quote("can't"), "'can'\"'\"'t'"); } + #[cfg(not(windows))] #[test] fn shell_command_script_marks_command_output_after_startup_noise() { let script = shell_command_script("npm", &["list", "-g", "@dbx-app/mcp-server", "--json"]); @@ -318,6 +325,7 @@ mod tests { assert!(script.contains("'@dbx-app/mcp-server'")); } + #[cfg(not(windows))] #[test] fn bash_login_script_sources_profile_and_rc_files() { let script = bash_login_script("node --version"); diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index e03052b97..73acaaed4 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -18,6 +18,8 @@ pub mod keychain; pub mod mcp; pub mod mcp_bridge; pub mod mongo_cmd; +#[cfg(feature = "mq-admin")] +pub mod mq_cmd; pub mod plugins; pub mod query; pub mod query_cancel; diff --git a/src-tauri/src/commands/mq_cmd.rs b/src-tauri/src/commands/mq_cmd.rs new file mode 100644 index 000000000..ae99a9ee3 --- /dev/null +++ b/src-tauri/src/commands/mq_cmd.rs @@ -0,0 +1,448 @@ +//! Desktop (Tauri) commands for message queue admin operations. +//! +//! These are thin wrappers around `dbx_core::mq::service::*_core` functions, +//! with read-only protection (`ensure_connection_writable`) for mutating calls. + +use std::sync::Arc; + +use tauri::State; + +use crate::commands::connection::{ensure_connection_writable, AppState}; + +// ---- Test connection ---- + +#[tauri::command] +pub async fn mq_test_connection( + state: State<'_, Arc>, + connection_id: String, +) -> Result { + dbx_core::mq::service::mq_test_connection_core(&state, &connection_id).await +} + +// ---- Tenants ---- + +#[tauri::command] +pub async fn mq_list_tenants( + state: State<'_, Arc>, + connection_id: String, +) -> Result, String> { + dbx_core::mq::service::mq_list_tenants_core(&state, &connection_id).await +} + +#[tauri::command] +pub async fn mq_get_tenant( + state: State<'_, Arc>, + connection_id: String, + name: String, +) -> Result { + dbx_core::mq::service::mq_get_tenant_core(&state, &connection_id, &name).await +} + +#[tauri::command] +pub async fn mq_create_tenant( + state: State<'_, Arc>, + connection_id: String, + name: String, + config: dbx_core::mq::TenantConfig, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Create tenant").await?; + dbx_core::mq::service::mq_create_tenant_core(&state, &connection_id, &name, config).await +} + +#[tauri::command] +pub async fn mq_update_tenant( + state: State<'_, Arc>, + connection_id: String, + name: String, + config: dbx_core::mq::TenantConfig, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Update tenant").await?; + dbx_core::mq::service::mq_update_tenant_core(&state, &connection_id, &name, config).await +} + +#[tauri::command] +pub async fn mq_delete_tenant( + state: State<'_, Arc>, + connection_id: String, + name: String, + force: bool, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Delete tenant").await?; + dbx_core::mq::service::mq_delete_tenant_core(&state, &connection_id, &name, force).await +} + +// ---- Namespaces ---- + +#[tauri::command] +pub async fn mq_list_namespaces( + state: State<'_, Arc>, + connection_id: String, + tenant: String, +) -> Result, String> { + dbx_core::mq::service::mq_list_namespaces_core(&state, &connection_id, &tenant).await +} + +#[tauri::command] +pub async fn mq_create_namespace( + state: State<'_, Arc>, + connection_id: String, + ns: dbx_core::mq::NamespaceRef, + config: dbx_core::mq::NamespaceConfig, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Create namespace").await?; + dbx_core::mq::service::mq_create_namespace_core(&state, &connection_id, ns, config).await +} + +#[tauri::command] +pub async fn mq_delete_namespace( + state: State<'_, Arc>, + connection_id: String, + ns: dbx_core::mq::NamespaceRef, + force: bool, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Delete namespace").await?; + dbx_core::mq::service::mq_delete_namespace_core(&state, &connection_id, ns, force).await +} + +#[tauri::command] +pub async fn mq_get_namespace_policies( + state: State<'_, Arc>, + connection_id: String, + ns: dbx_core::mq::NamespaceRef, +) -> Result { + dbx_core::mq::service::mq_get_namespace_policies_core(&state, &connection_id, ns).await +} + +// ---- Topics ---- + +#[tauri::command] +pub async fn mq_list_topics( + state: State<'_, Arc>, + connection_id: String, + ns: dbx_core::mq::NamespaceRef, + opts: dbx_core::mq::ListTopicsOpts, +) -> Result, String> { + dbx_core::mq::service::mq_list_topics_core(&state, &connection_id, ns, opts).await +} + +#[tauri::command] +pub async fn mq_create_topic( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + partitions: Option, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Create topic").await?; + dbx_core::mq::service::mq_create_topic_core(&state, &connection_id, topic, partitions).await +} + +#[tauri::command] +pub async fn mq_delete_topic( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + force: bool, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Delete topic").await?; + dbx_core::mq::service::mq_delete_topic_core(&state, &connection_id, topic, force).await +} + +#[tauri::command] +pub async fn mq_update_partitions( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + partitions: u32, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Update partitions").await?; + dbx_core::mq::service::mq_update_partitions_core(&state, &connection_id, topic, partitions).await +} + +#[tauri::command] +pub async fn mq_get_topic_stats( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, +) -> Result { + dbx_core::mq::service::mq_get_topic_stats_core(&state, &connection_id, topic).await +} + +#[tauri::command] +pub async fn mq_get_topic_internal_stats( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, +) -> Result { + dbx_core::mq::service::mq_get_topic_internal_stats_core(&state, &connection_id, topic).await +} + +// ---- Subscriptions ---- + +#[tauri::command] +pub async fn mq_list_subscriptions( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, +) -> Result, String> { + dbx_core::mq::service::mq_list_subscriptions_core(&state, &connection_id, topic).await +} + +#[tauri::command] +pub async fn mq_create_subscription( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + pos: dbx_core::mq::ResetPosition, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Create subscription").await?; + dbx_core::mq::service::mq_create_subscription_core(&state, &connection_id, topic, sub, pos).await +} + +#[tauri::command] +pub async fn mq_delete_subscription( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + force: bool, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Delete subscription").await?; + dbx_core::mq::service::mq_delete_subscription_core(&state, &connection_id, topic, sub, force).await +} + +#[tauri::command] +pub async fn mq_skip_messages( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + count: dbx_core::mq::SkipCount, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Skip messages").await?; + dbx_core::mq::service::mq_skip_messages_core(&state, &connection_id, topic, sub, count).await +} + +#[tauri::command] +pub async fn mq_reset_cursor( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + pos: dbx_core::mq::ResetPosition, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Reset cursor").await?; + dbx_core::mq::service::mq_reset_cursor_core(&state, &connection_id, topic, sub, pos).await +} + +#[tauri::command] +pub async fn mq_clear_backlog( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Clear backlog").await?; + dbx_core::mq::service::mq_clear_backlog_core(&state, &connection_id, topic, sub).await +} + +#[tauri::command] +pub async fn mq_peek_messages( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + count: u32, +) -> Result, String> { + dbx_core::mq::service::mq_peek_messages_core(&state, &connection_id, topic, sub, count).await +} + +#[tauri::command] +pub async fn mq_expire_messages( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, + expire_seconds: i64, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Expire messages").await?; + dbx_core::mq::service::mq_expire_messages_core(&state, &connection_id, topic, sub, expire_seconds).await +} + +// ---- Producers / consumers ---- + +#[tauri::command] +pub async fn mq_list_producers( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, +) -> Result, String> { + dbx_core::mq::service::mq_list_producers_core(&state, &connection_id, topic).await +} + +#[tauri::command] +pub async fn mq_list_consumers( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: String, +) -> Result, String> { + dbx_core::mq::service::mq_list_consumers_core(&state, &connection_id, topic, sub).await +} + +#[tauri::command] +pub async fn mq_unload_topic( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Unload topic").await?; + dbx_core::mq::service::mq_unload_topic_core(&state, &connection_id, topic).await +} + +// ---- Rate limits / quotas / retention ---- + +#[tauri::command] +pub async fn mq_set_publish_rate( + state: State<'_, Arc>, + connection_id: String, + scope: dbx_core::mq::PolicyScope, + rate: dbx_core::mq::PublishRate, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Set publish rate").await?; + dbx_core::mq::service::mq_set_publish_rate_core(&state, &connection_id, scope, rate).await +} + +#[tauri::command] +pub async fn mq_set_dispatch_rate( + state: State<'_, Arc>, + connection_id: String, + scope: dbx_core::mq::PolicyScope, + rate: dbx_core::mq::DispatchRate, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Set dispatch rate").await?; + dbx_core::mq::service::mq_set_dispatch_rate_core(&state, &connection_id, scope, rate).await +} + +#[tauri::command] +pub async fn mq_set_subscribe_rate( + state: State<'_, Arc>, + connection_id: String, + scope: dbx_core::mq::PolicyScope, + rate: dbx_core::mq::SubscribeRate, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Set subscribe rate").await?; + dbx_core::mq::service::mq_set_subscribe_rate_core(&state, &connection_id, scope, rate).await +} + +#[tauri::command] +pub async fn mq_set_backlog_quota( + state: State<'_, Arc>, + connection_id: String, + scope: dbx_core::mq::PolicyScope, + quota: dbx_core::mq::BacklogQuota, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Set backlog quota").await?; + dbx_core::mq::service::mq_set_backlog_quota_core(&state, &connection_id, scope, quota).await +} + +#[tauri::command] +pub async fn mq_set_retention( + state: State<'_, Arc>, + connection_id: String, + scope: dbx_core::mq::PolicyScope, + retention: dbx_core::mq::RetentionPolicy, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Set retention").await?; + dbx_core::mq::service::mq_set_retention_core(&state, &connection_id, scope, retention).await +} + +#[tauri::command] +pub async fn mq_get_effective_policies( + state: State<'_, Arc>, + connection_id: String, + scope: dbx_core::mq::PolicyScope, +) -> Result { + dbx_core::mq::service::mq_get_effective_policies_core(&state, &connection_id, scope).await +} + +// ---- Permissions ---- + +#[tauri::command] +pub async fn mq_grant_permission( + state: State<'_, Arc>, + connection_id: String, + scope: dbx_core::mq::PolicyScope, + role: String, + actions: Vec, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Grant permission").await?; + dbx_core::mq::service::mq_grant_permission_core(&state, &connection_id, scope, role, actions).await +} + +#[tauri::command] +pub async fn mq_revoke_permission( + state: State<'_, Arc>, + connection_id: String, + scope: dbx_core::mq::PolicyScope, + role: String, +) -> Result<(), String> { + ensure_connection_writable(&state, &connection_id, "Revoke permission").await?; + dbx_core::mq::service::mq_revoke_permission_core(&state, &connection_id, scope, role).await +} + +#[tauri::command] +pub async fn mq_list_permissions( + state: State<'_, Arc>, + connection_id: String, + scope: dbx_core::mq::PolicyScope, +) -> Result { + dbx_core::mq::service::mq_list_permissions_core(&state, &connection_id, scope).await +} + +// ---- Client tokens ---- + +#[tauri::command] +pub async fn mq_issue_token( + state: State<'_, Arc>, + connection_id: String, + req: dbx_core::mq::MqTokenIssueRequest, +) -> Result { + ensure_connection_writable(&state, &connection_id, "Issue MQ token").await?; + dbx_core::mq::service::mq_issue_token_core(&state, &connection_id, req).await +} + +#[tauri::command] +pub async fn mq_list_token_records( + state: State<'_, Arc>, + connection_id: String, + subject: Option, +) -> Result, String> { + dbx_core::mq::service::mq_list_token_records_core(&state, &connection_id, subject).await +} + +// ---- Monitoring ---- + +#[tauri::command] +pub async fn mq_get_backlog( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, + sub: Option, +) -> Result { + dbx_core::mq::service::mq_get_backlog_core(&state, &connection_id, topic, sub).await +} + +// ---- Raw request (escape hatch) ---- + +#[tauri::command] +pub async fn mq_raw_request( + state: State<'_, Arc>, + connection_id: String, + req: dbx_core::mq::MqRawRequest, +) -> Result { + if req.is_mutating() { + ensure_connection_writable(&state, &connection_id, "MQ admin write").await?; + } + dbx_core::mq::service::mq_raw_request_core(&state, &connection_id, req).await +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 8a388ae00..f21a09009 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -512,6 +512,86 @@ pub fn run() { commands::mongo_cmd::mongo_update_documents, commands::mongo_cmd::mongo_delete_document, commands::mongo_cmd::mongo_delete_documents, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_test_connection, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_list_tenants, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_get_tenant, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_create_tenant, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_update_tenant, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_delete_tenant, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_list_namespaces, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_create_namespace, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_delete_namespace, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_get_namespace_policies, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_list_topics, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_create_topic, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_delete_topic, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_update_partitions, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_get_topic_stats, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_get_topic_internal_stats, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_list_subscriptions, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_create_subscription, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_delete_subscription, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_skip_messages, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_reset_cursor, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_clear_backlog, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_peek_messages, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_expire_messages, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_list_producers, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_list_consumers, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_unload_topic, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_set_publish_rate, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_set_dispatch_rate, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_set_subscribe_rate, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_set_backlog_quota, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_set_retention, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_get_effective_policies, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_grant_permission, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_revoke_permission, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_list_permissions, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_issue_token, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_list_token_records, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_get_backlog, + #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_raw_request, commands::history::save_history, commands::history::load_history, commands::history::clear_history,