diff --git a/agents/drivers/kafka/build.gradle b/agents/drivers/kafka/build.gradle index 63f5e38ab..cbea97ab7 100644 --- a/agents/drivers/kafka/build.gradle +++ b/agents/drivers/kafka/build.gradle @@ -1,6 +1,15 @@ dependencies { implementation 'com.google.code.gson:gson:2.12.1' implementation 'org.apache.kafka:kafka-clients:3.9.0' + implementation('org.apache.zookeeper:zookeeper:3.8.4') { + exclude group: 'log4j', module: 'log4j' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + // The agent stdout is a JSON-RPC transport; Logback's default appender + // writes Kafka/ZooKeeper logs to stdout and corrupts that protocol. + exclude group: 'ch.qos.logback', module: 'logback-classic' + exclude group: 'ch.qos.logback', module: 'logback-core' + } + implementation 'io.dropwizard.metrics:metrics-core:4.1.12.1' runtimeOnly 'org.slf4j:slf4j-simple:1.7.36' } diff --git a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java index 2330d0ab2..d546bdbba 100644 --- a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java +++ b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java @@ -13,12 +13,21 @@ import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.client.ZKClientConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.InputStreamReader; +import java.io.PrintStream; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.*; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -29,9 +38,12 @@ import java.util.stream.Collectors; */ public final class KafkaAgent { + private static final PrintStream JSON_RPC_OUT = System.out; private static final Gson GSON = new GsonBuilder().serializeNulls().create(); private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000; private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000; + private static final int DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS = 10_000; + private static final String ZOOKEEPER_PROPERTY_PREFIX = "zookeeper."; private static final Set KERBEROS_SYSTEM_PROPERTY_KEYS = Set.of( "java.security.krb5.conf", "sun.security.krb5.debug", @@ -47,26 +59,40 @@ public final class KafkaAgent { private static AdminClient adminClient; private static KafkaProducer producer; + private static JsonObject activeConnection; private static volatile boolean shutdownRequested; private KafkaAgent() {} + private static Logger logger() { + // Initialize only after main redirects System.out, so any logging backend + // that defaults to stdout still cannot write into the JSON-RPC channel. + return LoggerHolder.INSTANCE; + } + + private static final class LoggerHolder { + private static final Logger INSTANCE = LoggerFactory.getLogger(KafkaAgent.class); + } + // ----------------------------------------------------------------------- // Entry point // ----------------------------------------------------------------------- public static void main(String[] args) throws Exception { + // Keep the original stdout exclusively for JSON-RPC. Redirect accidental + // System.out writes from dependencies to stderr so they cannot corrupt the protocol. + System.setOut(System.err); System.setProperty("org.slf4j.simpleLogger.logFile", "System.err"); - System.out.println("{\"ready\":true}"); - System.out.flush(); + JSON_RPC_OUT.println("{\"ready\":true}"); + JSON_RPC_OUT.flush(); BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); while (true) { String line = reader.readLine(); if (line == null) break; String response = handleRequest(line); - System.out.println(response); - System.out.flush(); + JSON_RPC_OUT.println(response); + JSON_RPC_OUT.flush(); if (shutdownRequested) { System.exit(0); } @@ -92,6 +118,7 @@ public final class KafkaAgent { Object result = dispatch(method, params); response.add("result", GSON.toJsonTree(result)); } catch (Exception e) { + logger().warn("Kafka Agent request failed: method={}, id={}", method, id, e); JsonObject error = new JsonObject(); error.addProperty("code", -1); error.addProperty("message", normalizeErrorMessage(e)); @@ -144,7 +171,7 @@ public final class KafkaAgent { } private static Object connect(JsonObject params) throws Exception { - JsonObject conn = connectionObject(params); + JsonObject conn = resolveBrokerConnection(connectionObject(params)); Map previousKerberosSystemProperties = applyKerberosSystemProperties(conn); AdminClient nextAdmin = null; KafkaProducer nextProducer = null; @@ -158,6 +185,7 @@ public final class KafkaAgent { applyKerberosSystemProperties(conn); adminClient = nextAdmin; producer = nextProducer; + activeConnection = conn.deepCopy(); return Collections.singletonMap("ok", true); } catch (Exception e) { if (nextAdmin != null) { @@ -172,7 +200,7 @@ public final class KafkaAgent { } private static Object testConnection(JsonObject params) throws Exception { - JsonObject conn = connectionObject(params); + JsonObject conn = resolveBrokerConnection(connectionObject(params)); Map previousKerberosSystemProperties = applyKerberosSystemProperties(conn); AdminClient probe = null; try { @@ -189,14 +217,11 @@ public final class KafkaAgent { probe.describeAcls(AclBindingFilter.ANY) .values().get(timeout, TimeUnit.MILLISECONDS); } catch (Exception aclEx) { - Throwable cause = aclEx; - while (cause != null) { - if (cause.getClass().getSimpleName().contains("SecurityDisabled") - || (cause.getMessage() != null && cause.getMessage().contains("No Authorizer"))) { - aclEnabled = false; - break; - } - cause = cause.getCause(); + if (isAclDisabledError(aclEx)) { + aclEnabled = false; + logger().debug("Kafka ACL support is disabled by the broker"); + } else { + logger().warn("Kafka ACL capability probe failed; leaving the capability enabled", aclEx); } } @@ -219,6 +244,18 @@ public final class KafkaAgent { } } + static boolean isAclDisabledError(Throwable error) { + Throwable cause = error; + while (cause != null) { + if (cause.getClass().getSimpleName().contains("SecurityDisabled") + || (cause.getMessage() != null && cause.getMessage().contains("No Authorizer"))) { + return true; + } + cause = cause.getCause(); + } + return false; + } + private static void closeClients() { if (adminClient != null) { adminClient.close(Duration.ofSeconds(5)); @@ -228,6 +265,7 @@ public final class KafkaAgent { producer.close(Duration.ofSeconds(5)); producer = null; } + activeConnection = null; restoreKerberosSystemProperties(BASELINE_KERBEROS_SYSTEM_PROPERTIES); } @@ -269,6 +307,160 @@ public final class KafkaAgent { return servers; } + static JsonObject resolveBrokerConnection(JsonObject conn) throws Exception { + String configured = stringOrEmpty(conn, "bootstrap_servers"); + if (configured.isBlank()) configured = stringOrEmpty(conn, "bootstrapServers"); + if (!configured.isBlank()) return conn; + + String connectString = stringOrEmpty(conn, "zookeeper_connect_string"); + if (connectString.isBlank()) connectString = stringOrEmpty(conn, "zookeeperServers"); + if (connectString.isBlank()) { + throw new IllegalArgumentException("bootstrap_servers or zookeeper_connect_string is required"); + } + + JsonObject resolved = conn.deepCopy(); + resolved.addProperty("bootstrap_servers", discoverBootstrapServers(connectString, securityProtocol(conn), conn)); + return resolved; + } + + private static String discoverBootstrapServers(String connectString, String securityProtocol, JsonObject conn) + throws Exception { + int sessionTimeout = intOrDefault(conn, "zookeeper_session_timeout_ms", DEFAULT_SESSION_TIMEOUT_MS); + int connectionTimeout = intOrDefault( + conn, + "zookeeper_connection_timeout_ms", + DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS + ); + CountDownLatch connected = new CountDownLatch(1); + ZooKeeper zooKeeper = new ZooKeeper(connectString, sessionTimeout, event -> { + if (event.getState() == Watcher.Event.KeeperState.SyncConnected) connected.countDown(); + }, zooKeeperClientConfig(conn)); + try { + if (!connected.await(connectionTimeout, TimeUnit.MILLISECONDS)) { + throw new IllegalStateException("Timed out connecting to ZooKeeper for Kafka broker discovery"); + } + + List brokerIds; + try { + brokerIds = new ArrayList<>(zooKeeper.getChildren("/brokers/ids", false)); + } catch (KeeperException.NoNodeException e) { + throw new IllegalStateException("ZooKeeper path /brokers/ids does not exist", e); + } + brokerIds.sort(KafkaAgent::compareBrokerIds); + + List registrations = new ArrayList<>(); + for (String brokerId : brokerIds) { + try { + byte[] data = zooKeeper.getData("/brokers/ids/" + brokerId, false, null); + registrations.add(JsonParser.parseString(new String(data, StandardCharsets.UTF_8)).getAsJsonObject()); + } catch (KeeperException.NoNodeException e) { + // Expected race: a broker may refresh its ephemeral node between list and read. + logger().debug("Kafka broker {} disappeared during ZooKeeper discovery", brokerId); + } catch (RuntimeException e) { + logger().warn("Skipping malformed ZooKeeper registration for Kafka broker {}", brokerId, e); + } + } + return brokerEndpoints(registrations, securityProtocol); + } finally { + zooKeeper.close(); + } + } + + static ZKClientConfig zooKeeperClientConfig(JsonObject conn) { + ZKClientConfig clientConfig = new ZKClientConfig(); + JsonObject properties = connectionProperties(conn); + if (properties == null) return clientConfig; + + for (Map.Entry entry : properties.entrySet()) { + if (entry.getKey().startsWith(ZOOKEEPER_PROPERTY_PREFIX) + && entry.getValue().isJsonPrimitive()) { + clientConfig.setProperty(entry.getKey(), entry.getValue().getAsString()); + } + } + return clientConfig; + } + + private static int compareBrokerIds(String left, String right) { + try { + return Integer.compare(Integer.parseInt(left), Integer.parseInt(right)); + } catch (NumberFormatException e) { + // Third-party registries may use non-numeric IDs; lexical ordering remains deterministic. + logger().debug("Sorting non-numeric Kafka broker IDs lexically: left={}, right={}", left, right); + return left.compareTo(right); + } + } + + static String brokerEndpoints(List registrations, String securityProtocol) { + String targetProtocol = securityProtocol == null || securityProtocol.isBlank() + ? "PLAINTEXT" + : securityProtocol.toUpperCase(Locale.ROOT); + Set addresses = new LinkedHashSet<>(); + + for (JsonObject registration : registrations) { + int addressCount = addresses.size(); + JsonObject protocolMap = registration.has("listener_security_protocol_map") + && registration.get("listener_security_protocol_map").isJsonObject() + ? registration.getAsJsonObject("listener_security_protocol_map") + : new JsonObject(); + JsonArray endpoints = registration.has("endpoints") && registration.get("endpoints").isJsonArray() + ? registration.getAsJsonArray("endpoints") + : new JsonArray(); + + for (JsonElement element : endpoints) { + if (!element.isJsonPrimitive() || !element.getAsJsonPrimitive().isString()) continue; + String endpoint = element.getAsString(); + int separator = endpoint.indexOf("://"); + if (separator <= 0) continue; + String listener = endpoint.substring(0, separator).toUpperCase(Locale.ROOT); + JsonElement mapped = protocolMap.get(listener); + String mappedProtocol = mapped != null && mapped.isJsonPrimitive() + ? mapped.getAsString().toUpperCase(Locale.ROOT) + : listener; + if (!targetProtocol.equals(mappedProtocol)) continue; + String address = endpointAddress(endpoint); + if (address != null) addresses.add(address); + } + + if (addresses.size() == addressCount && endpoints.size() == 0 && registration.has("host") && registration.has("port")) { + try { + String host = registration.get("host").getAsString().trim(); + int port = registration.get("port").getAsInt(); + if (!host.isEmpty() && port > 0 && port <= 65535) addresses.add(formatHostPort(host, port)); + } catch (RuntimeException e) { + logger().warn("Skipping malformed legacy Kafka broker registration", e); + } + } + } + + if (addresses.isEmpty()) { + throw new IllegalArgumentException("ZooKeeper did not return any usable Kafka broker endpoints"); + } + return String.join(",", addresses); + } + + private static String endpointAddress(String endpoint) { + try { + URI uri = URI.create(endpoint); + String host = uri.getHost(); + int port = uri.getPort(); + if (host == null || host.isBlank() || port <= 0 || port > 65535) return null; + return formatHostPort(host, port); + } catch (IllegalArgumentException e) { + logger().debug("Skipping malformed Kafka broker endpoint", e); + return null; + } + } + + private static String formatHostPort(String host, int port) { + return host.contains(":") && !host.startsWith("[") ? "[" + host + "]:" + port : host + ":" + port; + } + + private static String securityProtocol(JsonObject conn) { + String protocol = stringOrEmpty(conn, "security_protocol"); + if (protocol.isBlank()) protocol = stringOrEmpty(conn, "securityProtocol"); + return protocol.isBlank() ? "PLAINTEXT" : protocol; + } + static void applySecurityProperties(JsonObject conn, Properties props) { String securityProtocol = stringOrEmpty(conn, "security_protocol"); if (securityProtocol.isBlank()) { @@ -353,6 +545,7 @@ public final class KafkaAgent { for (Map.Entry entry : properties.entrySet()) { if (entry.getValue().isJsonPrimitive()) { String key = entry.getKey(); + if (key.startsWith(ZOOKEEPER_PROPERTY_PREFIX)) continue; String value = entry.getValue().getAsString(); props.put(key, value); } @@ -578,11 +771,47 @@ public final class KafkaAgent { } ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, name); - admin.incrementalAlterConfigs(Collections.singletonMap(resource, ops)) - .all().get(timeout, TimeUnit.MILLISECONDS); + try { + admin.incrementalAlterConfigs(Collections.singletonMap(resource, ops)) + .all().get(timeout, TimeUnit.MILLISECONDS); + } catch (Exception e) { + if (!isUnsupportedVersionError(e)) throw e; + logger().info("Kafka broker does not support incrementalAlterConfigs; using legacy alterConfigs for topic {}", name); + Config current = admin.describeConfigs(Collections.singletonList(resource)) + .all().get(timeout, TimeUnit.MILLISECONDS).get(resource); + Map values = legacyTopicConfig(current, ops); + Config replacement = new Config(values.entrySet().stream() + .map(entry -> new ConfigEntry(entry.getKey(), entry.getValue())) + .collect(Collectors.toList())); + admin.alterConfigs(Collections.singletonMap(resource, replacement)) + .all().get(timeout, TimeUnit.MILLISECONDS); + } return Collections.singletonMap("ok", true); } + static Map legacyTopicConfig(Config current, List ops) { + Map values = new LinkedHashMap<>(); + for (ConfigEntry entry : current.entries()) { + boolean topicOverride = entry.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG + || entry.source() == ConfigEntry.ConfigSource.UNKNOWN; + if (topicOverride && !entry.isReadOnly() && !entry.isSensitive() && entry.value() != null) { + values.put(entry.name(), entry.value()); + } + } + + for (AlterConfigOp op : ops) { + String key = op.configEntry().name(); + switch (op.opType()) { + case SET -> values.put(key, op.configEntry().value()); + case DELETE -> values.remove(key); + case APPEND, SUBTRACT -> throw new IllegalArgumentException( + "Kafka broker does not support " + op.opType() + " config operations through the legacy alterConfigs API" + ); + } + } + return values; + } + // ----------------------------------------------------------------------- // Consumer groups // ----------------------------------------------------------------------- @@ -698,6 +927,7 @@ public final class KafkaAgent { return Collections.singletonMap("producers", new ArrayList<>(byProducer.values())); } catch (Exception e) { if (isUnsupportedVersionError(e)) { + logger().info("Kafka broker does not support describeProducers; returning an empty producer list"); return Collections.singletonMap("producers", Collections.emptyList()); } throw e; @@ -797,26 +1027,11 @@ public final class KafkaAgent { Long offset = longOrNull(params, "offset"); int count = Math.max(1, intOrDefault(params, "count", 10)); - // Build a temporary consumer for peeking (no commit) - Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - adminClient != null ? adminClient.describeCluster().clusterId() - .get(5, TimeUnit.SECONDS) : "localhost:9092"); - // Reuse the admin's bootstrap servers - JsonObject conn = params.has("connection") && params.get("connection").isJsonObject() - ? params.getAsJsonObject("connection") : null; - if (conn != null) { - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn)); - applyConnectionProperties(conn, props); + JsonObject conn = activeConnection; + if (conn == null) { + throw new IllegalStateException("Kafka Agent is not connected"); } - props.put(ConsumerConfig.GROUP_ID_CONFIG, "dbx-peek-" + UUID.randomUUID()); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - "org.apache.kafka.common.serialization.StringDeserializer"); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none"); - props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, count); + Properties props = peekConsumerProperties(conn, count); try (KafkaConsumer consumer = new KafkaConsumer<>(props)) { List candidatePartitions = resolvePeekPartitions(consumer, topic, partition); @@ -872,6 +1087,21 @@ public final class KafkaAgent { } } + static Properties peekConsumerProperties(JsonObject conn, int count) { + Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn)); + applyConnectionProperties(conn, props); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "dbx-peek-" + UUID.randomUUID()); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringDeserializer"); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none"); + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, count); + return props; + } + /** * Poll until {@code count} messages are collected, every assigned partition has reached its * end offset, or {@code deadlineNs} expires. Empty polls retry until caught-up or deadline — @@ -1335,15 +1565,10 @@ public final class KafkaAgent { } private static String tryDecodeUtf8(byte[] bytes) { - try { - String text = new String(bytes, StandardCharsets.UTF_8); - // Verify round-trip - byte[] reEncoded = text.getBytes(StandardCharsets.UTF_8); - if (Arrays.equals(bytes, reEncoded)) { - return text; - } - } catch (Exception ignored) {} - return null; + String text = new String(bytes, StandardCharsets.UTF_8); + // Replacement characters change the bytes on round-trip, identifying invalid UTF-8 without exceptions. + byte[] reEncoded = text.getBytes(StandardCharsets.UTF_8); + return Arrays.equals(bytes, reEncoded) ? text : null; } static Long normalizePeekOffset(long requestedOffset, long beginningOffset, long endOffset) { diff --git a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java index 6ebe51311..c76cfd947 100644 --- a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java +++ b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java @@ -1,24 +1,282 @@ package com.dbx.agent.kafka; +import com.google.gson.JsonObject; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.gson.JsonParser; +import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.admin.AlterConfigOp; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.common.TopicPartition; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; class KafkaAgentTest { + @TempDir + Path tempDir; + + @Test + void resolvesBootstrapServersFromKafka11ZooKeeperRegistrationWithChroot() throws Exception { + Path snapshots = Files.createDirectory(tempDir.resolve("snapshots")); + Path logs = Files.createDirectory(tempDir.resolve("logs")); + ZooKeeperServer server = new ZooKeeperServer(snapshots.toFile(), logs.toFile(), 2_000); + NIOServerCnxnFactory factory = new NIOServerCnxnFactory(); + factory.configure(new InetSocketAddress("127.0.0.1", 0), 10); + factory.startup(server); + + ZooKeeper client = null; + String previousSaslSetting = System.getProperty("zookeeper.sasl.client"); + try { + CountDownLatch connected = new CountDownLatch(1); + System.setProperty("zookeeper.sasl.client", "false"); + client = new ZooKeeper("127.0.0.1:" + factory.getLocalPort(), 5_000, event -> { + if (event.getState() == Watcher.Event.KeeperState.SyncConnected) connected.countDown(); + }); + assertTrue(connected.await(5, TimeUnit.SECONDS)); + client.create("/kafka", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + client.create("/kafka/brokers", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + client.create("/kafka/brokers/ids", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + client.create( + "/kafka/brokers/ids/0", + "{\"listener_security_protocol_map\":{\"PLAINTEXT\":\"PLAINTEXT\"},\"endpoints\":[\"PLAINTEXT://legacy-broker:9092\"]}".getBytes(StandardCharsets.UTF_8), + ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL + ); + + JsonObject connection = new JsonObject(); + connection.addProperty("zookeeper_connect_string", "127.0.0.1:" + factory.getLocalPort() + "/kafka"); + connection.addProperty("security_protocol", "PLAINTEXT"); + connection.addProperty("zookeeper_connection_timeout_ms", 5_000); + + JsonObject resolved = KafkaAgent.resolveBrokerConnection(connection); + + assertEquals("legacy-broker:9092", resolved.get("bootstrap_servers").getAsString()); + } finally { + if (client != null) client.close(); + factory.shutdown(); + server.shutdown(); + server.getTxnLogFactory().close(); + if (previousSaslSetting == null) { + System.clearProperty("zookeeper.sasl.client"); + } else { + System.setProperty("zookeeper.sasl.client", previousSaslSetting); + } + } + } + + @Test + void zooKeeperClientConfigPreservesSaslAndTlsSystemDefaults() { + Map previous = preserveSystemProperties( + "zookeeper.sasl.client", + "zookeeper.sasl.clientconfig", + "zookeeper.client.secure", + "zookeeper.clientCnxnSocket", + "zookeeper.ssl.trustStore.location", + "java.security.auth.login.config" + ); + try { + System.setProperty("zookeeper.sasl.client", "true"); + System.setProperty("zookeeper.sasl.clientconfig", "DbxZooKeeperClient"); + System.setProperty("zookeeper.client.secure", "true"); + System.setProperty("zookeeper.clientCnxnSocket", "org.apache.zookeeper.ClientCnxnSocketNetty"); + System.setProperty("zookeeper.ssl.trustStore.location", "/etc/dbx/zookeeper-truststore.p12"); + System.setProperty("java.security.auth.login.config", "/etc/dbx/zookeeper-jaas.conf"); + + ZKClientConfig config = KafkaAgent.zooKeeperClientConfig(new JsonObject()); + + assertTrue(config.isSaslClientEnabled()); + assertEquals("DbxZooKeeperClient", config.getProperty("zookeeper.sasl.clientconfig")); + assertEquals("true", config.getProperty("zookeeper.client.secure")); + assertEquals( + "org.apache.zookeeper.ClientCnxnSocketNetty", + config.getProperty("zookeeper.clientCnxnSocket") + ); + assertEquals( + "/etc/dbx/zookeeper-truststore.p12", + config.getProperty("zookeeper.ssl.trustStore.location") + ); + assertEquals("/etc/dbx/zookeeper-jaas.conf", config.getJaasConfKey()); + } finally { + restoreSystemProperties(previous); + } + } + + @Test + void zooKeeperClientConfigAppliesPerConnectionSaslAndTlsOverridesWithoutChangingJvmState() { + Map previous = preserveSystemProperties( + "zookeeper.sasl.client", + "zookeeper.sasl.clientconfig", + "zookeeper.client.secure", + "zookeeper.clientCnxnSocket", + "zookeeper.ssl.keyStore.location" + ); + try { + System.setProperty("zookeeper.sasl.client", "false"); + System.setProperty("zookeeper.client.secure", "false"); + + JsonObject properties = new JsonObject(); + properties.addProperty("zookeeper.sasl.client", "true"); + properties.addProperty("zookeeper.sasl.clientconfig", "DbxZooKeeperClient"); + properties.addProperty("zookeeper.client.secure", "true"); + properties.addProperty("zookeeper.clientCnxnSocket", "org.apache.zookeeper.ClientCnxnSocketNetty"); + properties.addProperty("zookeeper.ssl.keyStore.location", "/etc/dbx/zookeeper-keystore.p12"); + properties.addProperty("security.protocol", "SASL_SSL"); + JsonObject connection = new JsonObject(); + connection.add("properties", properties); + + ZKClientConfig config = KafkaAgent.zooKeeperClientConfig(connection); + + assertTrue(config.isSaslClientEnabled()); + assertEquals("DbxZooKeeperClient", config.getProperty("zookeeper.sasl.clientconfig")); + assertEquals("true", config.getProperty("zookeeper.client.secure")); + assertEquals( + "org.apache.zookeeper.ClientCnxnSocketNetty", + config.getProperty("zookeeper.clientCnxnSocket") + ); + assertEquals( + "/etc/dbx/zookeeper-keystore.p12", + config.getProperty("zookeeper.ssl.keyStore.location") + ); + assertNull(config.getProperty("security.protocol")); + assertEquals("false", System.getProperty("zookeeper.sasl.client")); + assertEquals("false", System.getProperty("zookeeper.client.secure")); + } finally { + restoreSystemProperties(previous); + } + } + + @Test + void brokerEndpointsUseListenerSecurityProtocolMapForNamedListenersAndKeepBrokerOrder() { + List registrations = Arrays.asList( + broker("{\"listener_security_protocol_map\":{\"INTERNAL\":\"PLAINTEXT\",\"CLIENT\":\"SASL_SSL\"},\"endpoints\":[\"INTERNAL://broker-2:9092\",\"CLIENT://public-2:9093\"]}"), + broker("{\"listener_security_protocol_map\":{\"INTERNAL\":\"PLAINTEXT\",\"CLIENT\":\"SASL_SSL\"},\"endpoints\":[\"CLIENT://public-1:9093\",\"INTERNAL://broker-1:9092\"]}") + ); + + assertEquals("public-2:9093,public-1:9093", KafkaAgent.brokerEndpoints(registrations, "SASL_SSL")); + } + + @Test + void kafkaClientPropertiesExcludeZooKeeperSecuritySettings() { + JsonObject properties = new JsonObject(); + properties.addProperty("client.id", "dbx"); + properties.addProperty("zookeeper.sasl.client", "true"); + properties.addProperty("zookeeper.ssl.trustStore.password", "secret"); + JsonObject connection = new JsonObject(); + connection.add("properties", properties); + + Properties kafkaProperties = new Properties(); + KafkaAgent.applyConnectionProperties(connection, kafkaProperties); + + assertEquals("dbx", kafkaProperties.getProperty("client.id")); + assertNull(kafkaProperties.getProperty("zookeeper.sasl.client")); + assertNull(kafkaProperties.getProperty("zookeeper.ssl.trustStore.password")); + } + + @Test + void brokerEndpointsFallBackToLegacyHostAndPort() { + assertEquals("legacy-broker:9092", KafkaAgent.brokerEndpoints( + Collections.singletonList(broker("{\"host\":\"legacy-broker\",\"port\":9092}")), "PLAINTEXT")); + } + + @Test + void brokerEndpointsSkipMalformedRegistrationWhenAnotherBrokerIsUsable() { + assertEquals("healthy-broker:9092", KafkaAgent.brokerEndpoints(Arrays.asList( + broker("{\"host\":\"broken\",\"port\":\"not-a-port\"}"), + broker("{\"host\":\"healthy-broker\",\"port\":9092}") + ), "PLAINTEXT")); + } + + @Test + void brokerEndpointsRejectRegistrationsWithoutUsableAddresses() { + var error = org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> KafkaAgent.brokerEndpoints(Collections.singletonList(broker("{\"endpoints\":[]}")), "PLAINTEXT")); + assertTrue(error.getMessage().contains("usable Kafka broker endpoints")); + } + + @Test + void peekConsumerPropertiesReuseResolvedConnection() { + JsonObject resolved = new JsonObject(); + resolved.addProperty("bootstrap_servers", "legacy-broker:9092"); + resolved.addProperty("security_protocol", "PLAINTEXT"); + + Properties properties = KafkaAgent.peekConsumerProperties(resolved, 25); + + assertEquals("legacy-broker:9092", properties.getProperty("bootstrap.servers")); + assertEquals(25, properties.get("max.poll.records")); + } + + @Test + void aclDisabledDetectionOnlyAcceptsKnownAuthorizerErrors() { + Exception disabled = new RuntimeException( + "ACL probe failed", + new IllegalStateException("No Authorizer is configured on the broker") + ); + + assertTrue(KafkaAgent.isAclDisabledError(disabled)); + assertFalse(KafkaAgent.isAclDisabledError(new RuntimeException("Timed out waiting for broker response"))); + } + + @Test + void legacyTopicConfigAppliesSetAndDeleteWithoutLosingExistingOverrides() { + Config current = new Config(Arrays.asList( + new ConfigEntry("cleanup.policy", "delete"), + new ConfigEntry("retention.ms", "60000"), + new ConfigEntry( + "segment.bytes", + "1073741824", + ConfigEntry.ConfigSource.DYNAMIC_BROKER_CONFIG, + false, + false, + Collections.emptyList(), + ConfigEntry.ConfigType.LONG, + null + ) + )); + List ops = Arrays.asList( + new AlterConfigOp(new ConfigEntry("retention.ms", "120000"), AlterConfigOp.OpType.SET), + new AlterConfigOp(new ConfigEntry("cleanup.policy", null), AlterConfigOp.OpType.DELETE) + ); + + Map merged = KafkaAgent.legacyTopicConfig(current, ops); + + assertEquals(Collections.singletonMap("retention.ms", "120000"), merged); + } + + @Test + void legacyTopicConfigRejectsAppendAndSubtractOperations() { + Config current = new Config(Collections.singletonList(new ConfigEntry("cleanup.policy", "delete"))); + AlterConfigOp append = new AlterConfigOp(new ConfigEntry("cleanup.policy", "compact"), AlterConfigOp.OpType.APPEND); + + var error = org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> KafkaAgent.legacyTopicConfig(current, Collections.singletonList(append))); + assertTrue(error.getMessage().contains("APPEND")); + } @Test void normalizesPeekOffsetToEarliestAvailableOffset() { assertEquals(5L, KafkaAgent.normalizePeekOffset(0, 5, 10)); @@ -241,4 +499,24 @@ class KafkaAgentTest { } } } + + private static JsonObject broker(String json) { + return JsonParser.parseString(json).getAsJsonObject(); + } + + private static Map preserveSystemProperties(String... keys) { + Map previous = new HashMap<>(); + for (String key : keys) previous.put(key, System.getProperty(key)); + return previous; + } + + private static void restoreSystemProperties(Map properties) { + for (Map.Entry entry : properties.entrySet()) { + if (entry.getValue() == null) { + System.clearProperty(entry.getKey()); + } else { + System.setProperty(entry.getKey(), entry.getValue()); + } + } + } } diff --git a/apps/desktop/src/components/connection/ConnectionDialog.vue b/apps/desktop/src/components/connection/ConnectionDialog.vue index 477154324..180cbcc5e 100644 --- a/apps/desktop/src/components/connection/ConnectionDialog.vue +++ b/apps/desktop/src/components/connection/ConnectionDialog.vue @@ -46,7 +46,7 @@ import { SQLITE_DATABASE_FILE_EXTENSIONS } from "@/lib/database/databaseFileDete import { connectionAttemptOriginalErrorMessage, connectionAttemptTimeoutMessage, connectionAttemptTimeoutMs } from "@/lib/connection/connectionAttemptTimeout"; import { appendConnectionErrorHints, isJdbcMissingRuntimeDependencyError } from "@/lib/connection/connectionErrorHints"; import { postgresTlsModeForForm } from "@/lib/connection/postgresTlsMode"; -import { normalizeKafkaBootstrapServers } from "@/lib/connection/kafkaBootstrapServers"; +import { buildMqKafkaConnectionExtra, mqKafkaConnectionTarget, resolveMqKafkaConnectionSource, type MqKafkaConnectionSource } from "@/lib/connection/mqKafkaConnection"; import { assertCompleteDatabaseCategories, databaseSelectionForCategory } from "@/lib/connection/databaseCategoryOptions"; import { normalizeRocketmqNamesrvAddr } from "@/lib/connection/rocketmqNamesrv"; import { normalizeRabbitmqAddresses } from "@/lib/connection/rabbitmqAddresses"; @@ -517,11 +517,13 @@ const configTab = ref("connection"); const MQ_KAFKA_SECURITY_PROTOCOL_AUTO = "__auto"; const mqAdminUrl = ref("http://127.0.0.1:8080"); const mqSystemKind = ref("pulsar"); +const mqKafkaConnectionSource = ref("bootstrap"); const mqRocketmqNamesrvAddr = ref("127.0.0.1:9876"); const mqRocketmqClusterName = ref(""); const mqRabbitmqAddresses = ref("127.0.0.1:5672"); const mqRabbitmqVirtualHost = ref("/"); const mqKafkaBootstrapServers = ref("127.0.0.1:9092"); +const mqKafkaZooKeeperServers = ref(""); const mqKafkaSecurityProtocol = ref(MQ_KAFKA_SECURITY_PROTOCOL_AUTO); const mqKafkaSaslMechanism = ref("PLAIN"); const mqKafkaKerberosPrincipal = ref(""); @@ -575,6 +577,10 @@ const mqKafkaSecurityProtocolOptions = computed(() => [ { value: "SASL_PLAINTEXT", label: "SASL_PLAINTEXT" }, { value: "SASL_SSL", label: "SASL_SSL" }, ]); +const mqKafkaConnectionSourceOptions = computed(() => [ + { value: "bootstrap" as const, label: t("connection.mqKafkaConnectionSourceBootstrap") }, + { value: "zookeeper" as const, label: t("connection.mqKafkaConnectionSourceZooKeeper") }, +]); const mqKafkaSaslMechanismOptions = [ { value: "PLAIN", label: "PLAIN" }, { value: "SCRAM-SHA-256", label: "SCRAM-SHA-256" }, @@ -892,7 +898,9 @@ function resetMqFields(config?: Partial) { mqSystemKind.value = systemKind; const storedAdminUrl = config?.adminUrl?.trim() || (config ? mqExtraString(config as Record, "admin_url").trim() : ""); mqAdminUrl.value = storedAdminUrl || (systemKind === "kafka" || systemKind === "rocketmq" || systemKind === "rabbitmq" ? "" : "http://127.0.0.1:8080"); + mqKafkaConnectionSource.value = resolveMqKafkaConnectionSource(extra); mqKafkaBootstrapServers.value = mqExtraString(extra, "bootstrapServers") || "127.0.0.1:9092"; + mqKafkaZooKeeperServers.value = mqExtraString(extra, "zookeeperServers"); mqRocketmqNamesrvAddr.value = mqExtraString(extra, "namesrvAddr") || mqExtraString(extra, "namesrv_addr") || "127.0.0.1:9876"; mqRocketmqClusterName.value = mqExtraString(extra, "clusterName") || mqExtraString(extra, "cluster_name"); mqRabbitmqAddresses.value = mqExtraString(extra, "addresses") || "127.0.0.1:5672"; @@ -969,7 +977,7 @@ watch(selectedType, () => { watch(mqSystemKind, (kind) => { if (kind === "kafka") { - if (!mqKafkaBootstrapServers.value.trim()) mqKafkaBootstrapServers.value = "127.0.0.1:9092"; + if (mqKafkaConnectionSource.value === "bootstrap" && !mqKafkaBootstrapServers.value.trim()) mqKafkaBootstrapServers.value = "127.0.0.1:9092"; if (!isMqAuthKindAllowedForSystem(kind, mqAuthKind.value)) mqAuthKind.value = "none"; return; } @@ -1108,9 +1116,14 @@ function buildMqTokenSigning() { function buildMqAdminConfig(): MqAdminConfig { const systemKind = mqSystemKind.value; if (systemKind === "kafka") { - const bootstrapServers = normalizeKafkaBootstrapServers(mqKafkaBootstrapServers.value); - const extra: Record = { bootstrapServers }; - const securityProtocol = mqKafkaSecurityProtocol.value === MQ_KAFKA_SECURITY_PROTOCOL_AUTO ? "" : mqKafkaSecurityProtocol.value.trim(); + const configuredSecurityProtocol = mqKafkaSecurityProtocol.value === MQ_KAFKA_SECURITY_PROTOCOL_AUTO ? "" : mqKafkaSecurityProtocol.value; + const extra: Record = buildMqKafkaConnectionExtra({ + connectionSource: mqKafkaConnectionSource.value, + bootstrapServers: mqKafkaBootstrapServers.value, + zookeeperServers: mqKafkaZooKeeperServers.value, + securityProtocol: configuredSecurityProtocol, + }); + const securityProtocol = mqExtraString(extra, "securityProtocol"); const saslMechanism = mqAuthKind.value === "kerberos" ? "GSSAPI" : mqKafkaSaslMechanism.value.trim(); const properties: Record = {}; if (securityProtocol) extra.securityProtocol = securityProtocol; @@ -1518,18 +1531,17 @@ function applyMqAdminUrl(config: LegacyConnectionConfig, adminUrl: string) { config.ssl = parsed.protocol === "https:"; } -function applyMqKafkaBootstrapServers(config: LegacyConnectionConfig, bootstrapServers: string, securityProtocol?: string) { - const first = normalizeKafkaBootstrapServers(bootstrapServers).split(",")[0]; - if (!first) throw new Error(t("connection.mqBootstrapServersRequired")); - let parsed: URL; - try { - parsed = new URL(`kafka://${first}`); - } catch { - throw new Error(t("connection.mqBootstrapServersInvalid")); - } - config.host = parsed.hostname; - config.port = Number(parsed.port) || 9092; - config.ssl = securityProtocol === "SSL" || securityProtocol === "SASL_SSL"; +function applyMqKafkaConnectionTarget(config: LegacyConnectionConfig, extra: Record) { + const source = resolveMqKafkaConnectionSource(extra); + const target = mqKafkaConnectionTarget({ + connectionSource: source, + bootstrapServers: mqExtraString(extra, "bootstrapServers"), + zookeeperServers: mqExtraString(extra, "zookeeperServers"), + securityProtocol: mqExtraString(extra, "securityProtocol"), + }); + config.host = target.host; + config.port = target.port; + config.ssl = target.ssl; } function applyMqRabbitmqAddresses(config: LegacyConnectionConfig, addresses: string) { @@ -2548,7 +2560,7 @@ const connectionLabelTopClass = `${connectionLabelClass} mt-2`; const connectionLabelSmallPaddedClass = `${connectionLabelClass} pt-2 text-xs`; const hasRequiredConnectionTarget = computed(() => { if (form.value.db_type === "mq") { - if (mqSystemKind.value === "kafka") return !!mqKafkaBootstrapServers.value.trim(); + if (mqSystemKind.value === "kafka") return mqKafkaConnectionSource.value === "zookeeper" ? !!mqKafkaZooKeeperServers.value.trim() : !!mqKafkaBootstrapServers.value.trim(); if (mqSystemKind.value === "rocketmq") return !!mqRocketmqNamesrvAddr.value.trim(); if (mqSystemKind.value === "rabbitmq") return !!mqRabbitmqAddresses.value.trim(); return !!mqAdminUrl.value.trim(); @@ -2881,7 +2893,7 @@ function connectionConfigForSubmit(id: string, generatedName = ""): ConnectionCo config.driver_label = MQ_DRIVER_LABELS[mqConfig.systemKind]; if (mqConfig.systemKind === "kafka") { const extra = mqExtraRecord(mqConfig); - applyMqKafkaBootstrapServers(config, mqExtraString(extra, "bootstrapServers"), mqExtraString(extra, "securityProtocol")); + applyMqKafkaConnectionTarget(config, extra); } else if (mqConfig.systemKind === "rocketmq") { const extra = mqExtraRecord(mqConfig); applyMqRocketmqNamesrv(config, mqExtraString(extra, "namesrvAddr") || mqExtraString(extra, "namesrv_addr")); @@ -4769,9 +4781,26 @@ function openExternalUrl(url: string) {