From 1317301bcdc5fc519b0e44ccf5ef2166019e1ea3 Mon Sep 17 00:00:00 2001 From: zipg Date: Thu, 9 Jul 2026 11:41:22 +0800 Subject: [PATCH] feat(kafka): improve topic management and Kerberos auth --- .../java/com/dbx/agent/kafka/KafkaAgent.java | 78 ++++++- .../com/dbx/agent/kafka/KafkaAgentTest.java | 108 +++++++++ .../connection/ConnectionDialog.vue | 96 +++++++- .../src/components/layout/AppTabBar.vue | 24 +- .../src/components/mq/MonitoringPanel.vue | 191 +++++++++++++++- .../src/components/mq/MqAdminConsole.vue | 2 +- .../src/components/mq/SendMessagePanel.vue | 19 +- .../desktop/src/components/mq/TopicsPanel.vue | 206 +++++++++++++----- apps/desktop/src/i18n/locales/en.ts | 53 +++++ apps/desktop/src/i18n/locales/es.ts | 53 +++++ apps/desktop/src/i18n/locales/it.ts | 53 +++++ apps/desktop/src/i18n/locales/ja.ts | 53 +++++ apps/desktop/src/i18n/locales/pt-BR.ts | 53 +++++ apps/desktop/src/i18n/locales/zh-CN.ts | 53 +++++ apps/desktop/src/i18n/locales/zh-TW.ts | 53 +++++ apps/desktop/src/lib/connection/mqAuth.ts | 22 ++ crates/dbx-core/src/mq/adapters/kafka.rs | 28 +++ packages/app-tests/mqAuth.test.ts | 32 +++ 18 files changed, 1096 insertions(+), 81 deletions(-) create mode 100644 apps/desktop/src/lib/connection/mqAuth.ts create mode 100644 packages/app-tests/mqAuth.test.ts diff --git a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java index b53bd3195..ed732e0ca 100644 --- a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java +++ b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java @@ -32,6 +32,13 @@ public final class KafkaAgent { private static final Gson GSON = new GsonBuilder().serializeNulls().create(); private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000; private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000; + private static final Set KERBEROS_SYSTEM_PROPERTY_KEYS = Set.of( + "java.security.krb5.conf", + "sun.security.krb5.debug", + "javax.security.auth.useSubjectCredsOnly" + ); + private static final Map BASELINE_KERBEROS_SYSTEM_PROPERTIES = + snapshotKerberosSystemProperties(); private static final List CAPABILITIES = Collections.unmodifiableList(Arrays.asList( "mq_connect", "mq_test_connection", "mq_topics", "mq_consumer_groups", @@ -138,30 +145,38 @@ public final class KafkaAgent { private static Object connect(JsonObject params) throws Exception { JsonObject conn = connectionObject(params); - AdminClient nextAdmin = buildAdminClient(conn); + Map previousKerberosSystemProperties = applyKerberosSystemProperties(conn); + AdminClient nextAdmin = null; KafkaProducer nextProducer = null; try { + nextAdmin = buildAdminClient(conn); // Verify connectivity nextAdmin.describeCluster().clusterId().get( intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS), TimeUnit.MILLISECONDS); nextProducer = buildProducer(conn); closeClients(); + applyKerberosSystemProperties(conn); adminClient = nextAdmin; producer = nextProducer; return Collections.singletonMap("ok", true); } catch (Exception e) { - nextAdmin.close(Duration.ofSeconds(5)); + if (nextAdmin != null) { + nextAdmin.close(Duration.ofSeconds(5)); + } if (nextProducer != null) { nextProducer.close(Duration.ofSeconds(5)); } + restoreKerberosSystemProperties(previousKerberosSystemProperties); throw e; } } private static Object testConnection(JsonObject params) throws Exception { JsonObject conn = connectionObject(params); - AdminClient probe = buildAdminClient(conn); + Map previousKerberosSystemProperties = applyKerberosSystemProperties(conn); + AdminClient probe = null; try { + probe = buildAdminClient(conn); int timeout = intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS); DescribeClusterResult cluster = probe.describeCluster(); String clusterId = cluster.clusterId().get(timeout, TimeUnit.MILLISECONDS); @@ -197,7 +212,10 @@ public final class KafkaAgent { result.put("brokers", brokerList); return result; } finally { - probe.close(Duration.ofSeconds(5)); + if (probe != null) { + probe.close(Duration.ofSeconds(5)); + } + restoreKerberosSystemProperties(previousKerberosSystemProperties); } } @@ -210,6 +228,7 @@ public final class KafkaAgent { producer.close(Duration.ofSeconds(5)); producer = null; } + restoreKerberosSystemProperties(BASELINE_KERBEROS_SYSTEM_PROPERTIES); } // ----------------------------------------------------------------------- @@ -333,12 +352,61 @@ public final class KafkaAgent { if (properties != null) { for (Map.Entry entry : properties.entrySet()) { if (entry.getValue().isJsonPrimitive()) { - props.put(entry.getKey(), entry.getValue().getAsString()); + String key = entry.getKey(); + String value = entry.getValue().getAsString(); + props.put(key, value); } } } } + static Map applyKerberosSystemProperties(JsonObject conn) { + Map previous = snapshotKerberosSystemProperties(); + JsonObject properties = connectionProperties(conn); + for (String key : KERBEROS_SYSTEM_PROPERTY_KEYS) { + String value = stringProperty(properties, key); + if (value == null || value.isBlank()) { + value = BASELINE_KERBEROS_SYSTEM_PROPERTIES.get(key); + } + setOrClearSystemProperty(key, value); + } + return previous; + } + + static void restoreKerberosSystemProperties(Map values) { + for (String key : KERBEROS_SYSTEM_PROPERTY_KEYS) { + setOrClearSystemProperty(key, values.get(key)); + } + } + + private static Map snapshotKerberosSystemProperties() { + Map values = new LinkedHashMap<>(); + for (String key : KERBEROS_SYSTEM_PROPERTY_KEYS) { + values.put(key, System.getProperty(key)); + } + return values; + } + + private static JsonObject connectionProperties(JsonObject conn) { + return conn.has("properties") && conn.get("properties").isJsonObject() + ? conn.getAsJsonObject("properties") : null; + } + + private static String stringProperty(JsonObject properties, String key) { + if (properties == null || !properties.has(key) || !properties.get(key).isJsonPrimitive()) { + return null; + } + return properties.get(key).getAsString(); + } + + private static void setOrClearSystemProperty(String key, String value) { + if (value == null || value.isBlank()) { + System.clearProperty(key); + } else { + System.setProperty(key, value); + } + } + // ----------------------------------------------------------------------- // Topic management // ----------------------------------------------------------------------- diff --git a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java index 876a78969..1b4ad46db 100644 --- a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java +++ b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java @@ -3,6 +3,9 @@ package com.dbx.agent.kafka; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import com.google.gson.JsonParser; +import java.util.Map; +import java.util.Properties; import org.junit.jupiter.api.Test; class KafkaAgentTest { @@ -30,4 +33,109 @@ class KafkaAgentTest { void returnsNoSeekOffsetWhenTopicHasNoReadableMessages() { assertNull(KafkaAgent.normalizePeekOffset(0, 5, 5)); } + + @Test + void appliesKerberosKafkaProperties() { + Properties props = new Properties(); + KafkaAgent.applyConnectionProperties(JsonParser.parseString(""" + { + "security_protocol": "SASL_SSL", + "sasl_mechanism": "GSSAPI", + "properties": { + "sasl.jaas.config": "com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true keyTab=\\"/tmp/user.keytab\\" principal=\\"user@EXAMPLE.COM\\";", + "sasl.kerberos.service.name": "kafka" + } + } + """).getAsJsonObject(), props); + + assertEquals("SASL_SSL", props.getProperty("security.protocol")); + assertEquals("GSSAPI", props.getProperty("sasl.mechanism")); + assertEquals("kafka", props.getProperty("sasl.kerberos.service.name")); + assertEquals( + "com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true keyTab=\"/tmp/user.keytab\" principal=\"user@EXAMPLE.COM\";", + props.getProperty("sasl.jaas.config") + ); + } + + @Test + void appliesAllowedKerberosSystemPropertiesFromConnectionProperties() { + Map previous = KafkaAgent.applyKerberosSystemProperties(JsonParser.parseString(""" + { + "properties": { + "java.security.krb5.conf": "/tmp/krb5.conf", + "sun.security.krb5.debug": "true", + "custom.system.property": "should-not-leak" + } + } + """).getAsJsonObject()); + try { + assertEquals("/tmp/krb5.conf", System.getProperty("java.security.krb5.conf")); + assertEquals("true", System.getProperty("sun.security.krb5.debug")); + assertNull(System.getProperty("custom.system.property")); + } finally { + KafkaAgent.restoreKerberosSystemProperties(previous); + } + } + + @Test + void clearsPreviousKerberosSystemPropertiesForNextConnection() { + String baseline = System.getProperty("java.security.krb5.conf"); + Map previous = KafkaAgent.applyKerberosSystemProperties(JsonParser.parseString(""" + { + "properties": { + "java.security.krb5.conf": "/tmp/cluster-a.krb5.conf" + } + } + """).getAsJsonObject()); + try { + assertEquals("/tmp/cluster-a.krb5.conf", System.getProperty("java.security.krb5.conf")); + + Map beforeSecondConnection = KafkaAgent.applyKerberosSystemProperties(JsonParser.parseString(""" + { + "properties": { + "sasl.kerberos.service.name": "kafka" + } + } + """).getAsJsonObject()); + try { + assertEquals(baseline, System.getProperty("java.security.krb5.conf")); + } finally { + KafkaAgent.restoreKerberosSystemProperties(beforeSecondConnection); + } + } finally { + KafkaAgent.restoreKerberosSystemProperties(previous); + } + } + + @Test + void restoresKerberosSystemPropertiesWhenTestConnectionClientConstructionFails() { + String previous = System.getProperty("java.security.krb5.conf"); + try { + String response = KafkaAgent.handleRequest(""" + { + "jsonrpc": "2.0", + "id": 42, + "method": "test_connection", + "params": { + "connection": { + "bootstrap_servers": "", + "properties": { + "java.security.krb5.conf": "/tmp/leaked-test-connection.krb5.conf" + } + } + } + } + """); + + assertEquals(-1, JsonParser.parseString(response).getAsJsonObject() + .getAsJsonObject("error").get("code").getAsInt()); + assertEquals(previous, System.getProperty("java.security.krb5.conf")); + } finally { + if (previous == null) { + System.clearProperty("java.security.krb5.conf"); + } else { + System.setProperty("java.security.krb5.conf", previous); + } + } + } } diff --git a/apps/desktop/src/components/connection/ConnectionDialog.vue b/apps/desktop/src/components/connection/ConnectionDialog.vue index 2eff0f04a..21516bffa 100644 --- a/apps/desktop/src/components/connection/ConnectionDialog.vue +++ b/apps/desktop/src/components/connection/ConnectionDialog.vue @@ -40,6 +40,7 @@ import { SQLITE_DATABASE_FILE_EXTENSIONS } from "@/lib/database/databaseFileDete import { connectionAttemptOriginalErrorMessage, connectionAttemptTimeoutMessage, connectionAttemptTimeoutMs } from "@/lib/connection/connectionAttemptTimeout"; import { appendConnectionErrorHints } from "@/lib/connection/connectionErrorHints"; import { normalizeKafkaBootstrapServers } from "@/lib/connection/kafkaBootstrapServers"; +import { detectMqUiAuthKind, isMqAuthKindAllowedForSystem, type MqUiAuthKind } from "@/lib/connection/mqAuth"; import { driverInstallProgressPercent, type DriverInstallProgress } from "@/lib/connection/driverInstallProgressUi"; import { ArrowLeft, ArrowDown, ArrowUp, CheckSquare, ChevronRight, CircleHelp, Copy, ExternalLink, FilePlus2, FolderOpen, GripVertical, Grid3X3, KeyRound, Link2, List, ListFilter, Loader2, Pencil, Pipette, Plus, Search, ShieldCheck, Square, Trash2 } from "@lucide/vue"; import { buildDraftVisibleDatabasesConnectionId, connectionCanChooseVisibleDatabases, initialVisibleDatabaseSelection, visibleDatabaseSelectionIsStale } from "@/lib/connection/connectionVisibleDatabases"; @@ -390,14 +391,17 @@ const dialogStep = ref("select"); const dbPickerView = ref("icon"); const dbSearchQuery = ref(""); const configTab = ref("connection"); -type MqAuthKind = MqAuth["kind"]; const MQ_KAFKA_SECURITY_PROTOCOL_AUTO = "__auto"; const mqAdminUrl = ref("http://127.0.0.1:8080"); const mqSystemKind = ref("pulsar"); const mqKafkaBootstrapServers = ref("127.0.0.1:9092"); const mqKafkaSecurityProtocol = ref(MQ_KAFKA_SECURITY_PROTOCOL_AUTO); const mqKafkaSaslMechanism = ref("PLAIN"); -const mqAuthKind = ref("none"); +const mqKafkaKerberosPrincipal = ref(""); +const mqKafkaKerberosKeytabPath = ref(""); +const mqKafkaKerberosServiceName = ref("kafka"); +const mqKafkaKrb5ConfPath = ref(""); +const mqAuthKind = ref("none"); const mqToken = ref(""); const mqBasicUsername = ref(""); const mqBasicPassword = ref(""); @@ -693,18 +697,50 @@ function mqExtraString(extra: Record, key: string): string { return typeof value === "string" ? value : ""; } +function mqExtraProperties(extra: Record): Record { + const value = extra.properties; + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function mqExtraPropertyString(extra: Record, key: string): string { + const value = mqExtraProperties(extra)[key]; + return typeof value === "string" ? value : ""; +} + +function jaasStringValue(value: string): string { + return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"'); +} + +function parseJaasStringProperty(value: string, key: string): string { + const escapedKey = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const match = value.match(new RegExp(`${escapedKey}\\s*=\\s*"((?:\\\\.|[^"\\\\])*)"`, "i")); + if (!match) return ""; + return match[1].replace(/\\(["\\])/g, "$1"); +} + function resetMqFields(config?: Partial) { const systemKind = config?.systemKind === "kafka" ? "kafka" : "pulsar"; const extra = mqExtraRecord(config); + const properties = mqExtraProperties(extra); + const jaasConfig = mqExtraPropertyString(extra, "sasl.jaas.config"); mqSystemKind.value = systemKind; mqAdminUrl.value = config?.adminUrl?.trim() || (systemKind === "kafka" ? "" : "http://127.0.0.1:8080"); mqKafkaBootstrapServers.value = mqExtraString(extra, "bootstrapServers") || "127.0.0.1:9092"; mqKafkaSecurityProtocol.value = mqExtraString(extra, "securityProtocol") || MQ_KAFKA_SECURITY_PROTOCOL_AUTO; mqKafkaSaslMechanism.value = mqExtraString(extra, "saslMechanism") || "PLAIN"; + mqKafkaKerberosPrincipal.value = parseJaasStringProperty(jaasConfig, "principal"); + mqKafkaKerberosKeytabPath.value = parseJaasStringProperty(jaasConfig, "keyTab"); + mqKafkaKerberosServiceName.value = typeof properties["sasl.kerberos.service.name"] === "string" ? properties["sasl.kerberos.service.name"] : "kafka"; + mqKafkaKrb5ConfPath.value = typeof properties["java.security.krb5.conf"] === "string" ? properties["java.security.krb5.conf"] : ""; mqTlsSkipVerify.value = !!config?.tlsSkipVerify; mqPinnedVersion.value = pinnedVersionToSelection(config?.pinnedVersion); const auth = (config?.auth || { kind: "none" }) as MqAuth; - mqAuthKind.value = systemKind === "kafka" && auth.kind !== "basic" ? "none" : auth.kind || "none"; + mqAuthKind.value = detectMqUiAuthKind({ + systemKind, + authKind: auth.kind, + saslMechanism: mqKafkaSaslMechanism.value, + jaasConfig, + }); mqToken.value = auth.token || ""; mqBasicUsername.value = auth.username || ""; mqBasicPassword.value = auth.password || ""; @@ -741,12 +777,18 @@ function hydrateMqFields(value: unknown) { watch(mqSystemKind, (kind) => { if (kind === "kafka") { if (!mqKafkaBootstrapServers.value.trim()) mqKafkaBootstrapServers.value = "127.0.0.1:9092"; - if (!["none", "basic"].includes(mqAuthKind.value)) mqAuthKind.value = "none"; + if (!isMqAuthKindAllowedForSystem(kind, mqAuthKind.value)) mqAuthKind.value = "none"; return; } if (!mqAdminUrl.value.trim()) mqAdminUrl.value = "http://127.0.0.1:8080"; }); +watch(mqAuthKind, (kind) => { + if (mqSystemKind.value === "kafka" && kind === "basic" && mqKafkaSaslMechanism.value.toUpperCase() === "GSSAPI") { + mqKafkaSaslMechanism.value = "PLAIN"; + } +}); + function resetNacosFields(config?: Partial) { nacosServerAddr.value = config?.serverAddr?.trim() || NACOS_DEFAULT_CONSOLE_URL; nacosNamespace.value = config?.namespace || ""; @@ -845,6 +887,12 @@ function buildMqAuth(): MqAuth { } } +function buildKafkaKerberosJaasConfig(): string { + const principal = requireMqField(mqKafkaKerberosPrincipal.value, t("connection.kafkaKerberosPrincipalRequired")); + const keytab = requireMqField(mqKafkaKerberosKeytabPath.value, t("connection.kafkaKerberosKeytabRequired")); + return `com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab="${jaasStringValue(keytab)}" principal="${jaasStringValue(principal)}";`; +} + function buildMqTokenSigning() { if (mqTokenSigningMode.value === "none") return undefined; return { @@ -857,11 +905,21 @@ function buildMqAdminConfig(): MqAdminConfig { const systemKind = mqSystemKind.value; if (systemKind === "kafka") { const bootstrapServers = normalizeKafkaBootstrapServers(mqKafkaBootstrapServers.value); - const extra: Record = { bootstrapServers }; + const extra: Record = { bootstrapServers }; const securityProtocol = mqKafkaSecurityProtocol.value === MQ_KAFKA_SECURITY_PROTOCOL_AUTO ? "" : mqKafkaSecurityProtocol.value.trim(); - const saslMechanism = mqKafkaSaslMechanism.value.trim(); + const saslMechanism = mqAuthKind.value === "kerberos" ? "GSSAPI" : mqKafkaSaslMechanism.value.trim(); + const properties: Record = {}; if (securityProtocol) extra.securityProtocol = securityProtocol; if (mqAuthKind.value === "basic" && saslMechanism) extra.saslMechanism = saslMechanism; + if (mqAuthKind.value === "kerberos") { + extra.saslMechanism = "GSSAPI"; + properties["sasl.jaas.config"] = buildKafkaKerberosJaasConfig(); + properties["sasl.kerberos.service.name"] = mqKafkaKerberosServiceName.value.trim() || "kafka"; + if (mqKafkaKrb5ConfPath.value.trim()) { + properties["java.security.krb5.conf"] = mqKafkaKrb5ConfPath.value.trim(); + } + } + if (Object.keys(properties).length) extra.properties = properties; return { systemKind: mqSystemKind.value, adminUrl: "", @@ -3908,6 +3966,7 @@ function openExternalUrl(url: string) { + @@ -3941,6 +4000,31 @@ function openExternalUrl(url: string) { +