diff --git a/agents/build.gradle b/agents/build.gradle index b3dff3914..5c9ca37da 100644 --- a/agents/build.gradle +++ b/agents/build.gradle @@ -4,7 +4,7 @@ plugins { def java8Projects = ['common', 'test-support'] as Set def infrastructureProjects = ['common', 'test-support'] as Set -def legacyStandaloneProjects = ['mongodb'] as Set +def legacyStandaloneProjects = ['mongodb', 'kafka'] as Set def agentProjects = subprojects.findAll { !infrastructureProjects.contains(it.name) } def jdbcAgentProjects = agentProjects.findAll { !legacyStandaloneProjects.contains(it.name) } diff --git a/agents/drivers/kafka/build.gradle b/agents/drivers/kafka/build.gradle new file mode 100644 index 000000000..63f5e38ab --- /dev/null +++ b/agents/drivers/kafka/build.gradle @@ -0,0 +1,12 @@ +dependencies { + implementation 'com.google.code.gson:gson:2.12.1' + implementation 'org.apache.kafka:kafka-clients:3.9.0' + runtimeOnly 'org.slf4j:slf4j-simple:1.7.36' +} + +tasks.named('shadowJar') { + mergeServiceFiles() + manifest { + attributes('Agent-Label': 'Apache Kafka', 'Main-Class': 'com.dbx.agent.kafka.KafkaAgent') + } +} diff --git a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java new file mode 100644 index 000000000..cd81b902d --- /dev/null +++ b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java @@ -0,0 +1,1095 @@ +package com.dbx.agent.kafka; + +import com.google.gson.*; +import org.apache.kafka.clients.admin.*; +import org.apache.kafka.clients.consumer.*; +import org.apache.kafka.clients.producer.*; +import org.apache.kafka.common.*; +import org.apache.kafka.common.acl.*; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.errors.*; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.resource.ResourcePatternFilter; +import org.apache.kafka.common.resource.ResourceType; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * Kafka admin agent for DBX. Communicates with the Rust bridge via JSON-RPC + * over stdin/stdout. Uses kafka-clients AdminClient for admin operations and + * KafkaProducer for message production. + */ +public final class KafkaAgent { + + private static final Gson GSON = new GsonBuilder().serializeNulls().create(); + private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000; + private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000; + + private static final List CAPABILITIES = Collections.unmodifiableList(Arrays.asList( + "mq_connect", "mq_test_connection", "mq_topics", "mq_consumer_groups", + "mq_messages", "mq_acl", "mq_config", "mq_monitoring" + )); + + private static AdminClient adminClient; + private static KafkaProducer producer; + private static volatile boolean shutdownRequested; + + private KafkaAgent() {} + + // ----------------------------------------------------------------------- + // Entry point + // ----------------------------------------------------------------------- + + public static void main(String[] args) throws Exception { + System.setProperty("org.slf4j.simpleLogger.logFile", "System.err"); + System.out.println("{\"ready\":true}"); + System.out.flush(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); + while (true) { + String line = reader.readLine(); + if (line == null) break; + String response = handleRequest(line); + System.out.println(response); + System.out.flush(); + if (shutdownRequested) { + System.exit(0); + } + } + } + + // ----------------------------------------------------------------------- + // JSON-RPC dispatch + // ----------------------------------------------------------------------- + + static String handleRequest(String line) { + JsonObject req = JsonParser.parseString(line).getAsJsonObject(); + JsonElement id = req.get("id"); + String method = req.get("method").getAsString(); + JsonObject params = req.has("params") && req.get("params").isJsonObject() + ? req.getAsJsonObject("params") : new JsonObject(); + + JsonObject response = new JsonObject(); + response.addProperty("jsonrpc", "2.0"); + response.add("id", id); + + try { + Object result = dispatch(method, params); + response.add("result", GSON.toJsonTree(result)); + } catch (Exception e) { + JsonObject error = new JsonObject(); + error.addProperty("code", -1); + error.addProperty("message", e.getMessage() == null ? "Unknown error" : e.getMessage()); + response.add("error", error); + } + return GSON.toJson(response); + } + + private static Object dispatch(String method, JsonObject params) throws Exception { + return switch (method) { + case "handshake" -> handshakeResult(); + case "connect" -> connect(params); + case "test_connection" -> testConnection(params); + case "disconnect" -> { closeClients(); yield Collections.singletonMap("ok", true); } + case "shutdown" -> { closeClients(); shutdownRequested = true; yield Collections.singletonMap("ok", true); } + // Topic management + case "mq_list_topics" -> listTopics(params); + case "mq_create_topic" -> createTopic(params); + case "mq_delete_topic" -> deleteTopic(params); + case "mq_update_partitions" -> updatePartitions(params); + case "mq_get_topic_stats" -> getTopicStats(params); + case "mq_get_topic_config" -> getTopicConfig(params); + case "mq_alter_topic_config" -> alterTopicConfig(params); + // Consumer groups + case "mq_list_consumer_groups" -> listConsumerGroups(params); + case "mq_describe_consumer_group" -> describeConsumerGroup(params); + case "mq_delete_consumer_group" -> deleteConsumerGroup(params); + case "mq_reset_consumer_group_offsets" -> resetConsumerGroupOffsets(params); + case "mq_list_producers" -> listProducers(params); + // Messages + case "mq_peek_messages" -> peekMessages(params); + case "mq_send_message" -> sendMessage(params); + // ACLs + case "mq_list_acls" -> listAcls(params); + case "mq_create_acls" -> createAcls(params); + case "mq_delete_acls" -> deleteAcls(params); + // Cluster / monitoring + case "mq_describe_cluster" -> describeCluster(params); + case "mq_get_consumer_lag" -> getConsumerLag(params); + default -> throw new IllegalArgumentException("Unknown method: " + method); + }; + } + + // ----------------------------------------------------------------------- + // Lifecycle + // ----------------------------------------------------------------------- + + private static Object handshakeResult() { + return new HandshakeResult(1, 1, CAPABILITIES); + } + + private static Object connect(JsonObject params) throws Exception { + JsonObject conn = connectionObject(params); + AdminClient nextAdmin = buildAdminClient(conn); + KafkaProducer nextProducer = null; + try { + // Verify connectivity + nextAdmin.describeCluster().clusterId().get( + intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS), TimeUnit.MILLISECONDS); + nextProducer = buildProducer(conn); + closeClients(); + adminClient = nextAdmin; + producer = nextProducer; + return Collections.singletonMap("ok", true); + } catch (Exception e) { + nextAdmin.close(Duration.ofSeconds(5)); + if (nextProducer != null) { + nextProducer.close(Duration.ofSeconds(5)); + } + throw e; + } + } + + private static Object testConnection(JsonObject params) throws Exception { + JsonObject conn = connectionObject(params); + AdminClient probe = buildAdminClient(conn); + try { + int timeout = intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS); + DescribeClusterResult cluster = probe.describeCluster(); + String clusterId = cluster.clusterId().get(timeout, TimeUnit.MILLISECONDS); + Node controller = cluster.controller().get(timeout, TimeUnit.MILLISECONDS); + Collection brokers = cluster.nodes().get(timeout, TimeUnit.MILLISECONDS); + + // Probe ACL support: try a describe operation and catch security errors. + boolean aclEnabled = true; + try { + probe.describeAcls(AclBindingFilter.ANY) + .values().get(timeout, TimeUnit.MILLISECONDS); + } catch (Exception aclEx) { + Throwable cause = aclEx; + while (cause != null) { + if (cause.getClass().getSimpleName().contains("SecurityDisabled") + || (cause.getMessage() != null && cause.getMessage().contains("No Authorizer"))) { + aclEnabled = false; + break; + } + cause = cause.getCause(); + } + } + + Map result = new LinkedHashMap<>(); + result.put("ok", true); + result.put("clusterId", clusterId); + result.put("controller", controller != null ? nodeToMap(controller) : null); + result.put("aclEnabled", aclEnabled); + List> brokerList = new ArrayList<>(); + for (Node node : brokers) { + brokerList.add(nodeToMap(node)); + } + result.put("brokers", brokerList); + return result; + } finally { + probe.close(Duration.ofSeconds(5)); + } + } + + private static void closeClients() { + if (adminClient != null) { + adminClient.close(Duration.ofSeconds(5)); + adminClient = null; + } + if (producer != null) { + producer.close(Duration.ofSeconds(5)); + producer = null; + } + } + + // ----------------------------------------------------------------------- + // Client builders + // ----------------------------------------------------------------------- + + static AdminClient buildAdminClient(JsonObject conn) { + Properties props = new Properties(); + props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn)); + props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, + intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS)); + props.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, + intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS)); + applyConnectionProperties(conn, props); + return AdminClient.create(props); + } + + private static KafkaProducer buildProducer(JsonObject conn) { + Properties props = new Properties(); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn)); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer"); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArraySerializer"); + props.put(ProducerConfig.ACKS_CONFIG, "all"); + applyConnectionProperties(conn, props); + return new KafkaProducer<>(props); + } + + private static String bootstrapServers(JsonObject conn) { + String servers = stringOrEmpty(conn, "bootstrap_servers"); + if (servers.isBlank()) { + servers = stringOrEmpty(conn, "bootstrapServers"); + } + if (servers.isBlank()) { + throw new IllegalArgumentException("bootstrap_servers is required"); + } + return servers; + } + + static void applySecurityProperties(JsonObject conn, Properties props) { + String securityProtocol = stringOrEmpty(conn, "security_protocol"); + if (securityProtocol.isBlank()) { + securityProtocol = stringOrEmpty(conn, "securityProtocol"); + } + if (!securityProtocol.isBlank()) { + props.put("security.protocol", securityProtocol); + } + + String saslMechanism = stringOrEmpty(conn, "sasl_mechanism"); + if (saslMechanism.isBlank()) { + saslMechanism = stringOrEmpty(conn, "saslMechanism"); + } + if (!saslMechanism.isBlank()) { + props.put("sasl.mechanism", saslMechanism); + } + + String saslUsername = stringOrEmpty(conn, "sasl_username"); + if (saslUsername.isBlank()) saslUsername = stringOrEmpty(conn, "saslUsername"); + String saslPassword = stringOrEmpty(conn, "sasl_password"); + if (saslPassword.isBlank()) saslPassword = stringOrEmpty(conn, "saslPassword"); + + if (!saslUsername.isBlank() && !saslMechanism.isBlank()) { + String jaasTemplate = switch (saslMechanism.toUpperCase()) { + case "PLAIN" -> "org.apache.kafka.common.security.plain.PlainLoginModule required " + + "username=\"%s\" password=\"%s\";"; + case "SCRAM-SHA-256", "SCRAM-SHA-512" -> + "org.apache.kafka.common.security.scram.ScramLoginModule required " + + "username=\"%s\" password=\"%s\";"; + default -> null; + }; + if (jaasTemplate != null) { + props.put("sasl.jaas.config", String.format(jaasTemplate, jaasValue(saslUsername), jaasValue(saslPassword))); + } + } + + JsonObject tls = conn.has("tls") && conn.get("tls").isJsonObject() + ? conn.getAsJsonObject("tls") : null; + boolean skipVerify = boolOrDefault(conn, "tls_skip_verify", false) + || boolOrDefault(conn, "tlsSkipVerify", false) + || (tls != null && boolOrDefault(tls, "skip_verify", false)); + if (skipVerify) { + props.put("ssl.endpoint.identification.algorithm", ""); + } + + // TLS properties + if (tls != null) { + String truststorePath = stringOrEmpty(tls, "truststore_path"); + if (!truststorePath.isBlank()) { + props.put("ssl.truststore.location", truststorePath); + String truststorePassword = stringOrEmpty(tls, "truststore_password"); + if (!truststorePassword.isBlank()) { + props.put("ssl.truststore.password", truststorePassword); + } + } + String keystorePath = stringOrEmpty(tls, "keystore_path"); + if (!keystorePath.isBlank()) { + props.put("ssl.keystore.location", keystorePath); + String keystorePassword = stringOrEmpty(tls, "keystore_password"); + if (!keystorePassword.isBlank()) { + props.put("ssl.keystore.password", keystorePassword); + } + } + } + } + + static void applyConnectionProperties(JsonObject conn, Properties props) { + applySecurityProperties(conn, props); + applyExtraProperties(conn, props); + } + + static String jaasValue(String value) { + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } + + @SuppressWarnings("unchecked") + private static void applyExtraProperties(JsonObject conn, Properties props) { + JsonObject properties = conn.has("properties") && conn.get("properties").isJsonObject() + ? conn.getAsJsonObject("properties") : null; + if (properties != null) { + for (Map.Entry entry : properties.entrySet()) { + if (entry.getValue().isJsonPrimitive()) { + props.put(entry.getKey(), entry.getValue().getAsString()); + } + } + } + } + + // ----------------------------------------------------------------------- + // Topic management + // ----------------------------------------------------------------------- + + private static Object listTopics(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + Set names = admin.listTopics(new ListTopicsOptions().timeoutMs(timeout)) + .names().get(timeout, TimeUnit.MILLISECONDS); + if (names.isEmpty()) { + return Collections.singletonMap("topics", Collections.emptyList()); + } + + Map descriptions = admin.describeTopics(names) + .allTopicNames().get(timeout, TimeUnit.MILLISECONDS); + + List> topics = new ArrayList<>(); + for (Map.Entry entry : descriptions.entrySet()) { + TopicDescription desc = entry.getValue(); + Map topic = new LinkedHashMap<>(); + topic.put("name", desc.name()); + topic.put("partitions", desc.partitions().size()); + topic.put("replicationFactor", desc.partitions().isEmpty() ? 0 + : desc.partitions().get(0).replicas().size()); + topic.put("internal", desc.isInternal()); + topics.add(topic); + } + topics.sort(Comparator.comparing(m -> (String) m.get("name"))); + return Collections.singletonMap("topics", topics); + } + + private static Object createTopic(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String name = stringOrEmpty(params, "name"); + int partitions = intOrDefault(params, "partitions", 1); + short replicationFactor = (short) intOrDefault(params, "replicationFactor", 1); + + NewTopic newTopic = new NewTopic(name, partitions, replicationFactor); + + // Optional configs + JsonObject configs = params.has("configs") && params.get("configs").isJsonObject() + ? params.getAsJsonObject("configs") : null; + if (configs != null) { + Map configMap = new HashMap<>(); + for (Map.Entry entry : configs.entrySet()) { + configMap.put(entry.getKey(), entry.getValue().getAsString()); + } + newTopic.configs(configMap); + } + + admin.createTopics(Collections.singletonList(newTopic)) + .all().get(timeout, TimeUnit.MILLISECONDS); + return Collections.singletonMap("ok", true); + } + + private static Object deleteTopic(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String name = stringOrEmpty(params, "name"); + admin.deleteTopics(Collections.singletonList(name)) + .all().get(timeout, TimeUnit.MILLISECONDS); + return Collections.singletonMap("ok", true); + } + + private static Object updatePartitions(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String name = stringOrEmpty(params, "name"); + int totalPartitions = intOrDefault(params, "totalPartitions", 1); + admin.createPartitions(Collections.singletonMap(name, NewPartitions.increaseTo(totalPartitions))) + .all().get(timeout, TimeUnit.MILLISECONDS); + return Collections.singletonMap("ok", true); + } + + private static Object getTopicStats(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String name = stringOrEmpty(params, "name"); + + TopicDescription desc = admin.describeTopics(Collections.singletonList(name)) + .allTopicNames().get(timeout, TimeUnit.MILLISECONDS).get(name); + + // Collect offsets for size estimation + Map endOffsets = new LinkedHashMap<>(); + Map beginOffsets = new LinkedHashMap<>(); + for (TopicPartitionInfo pi : desc.partitions()) { + TopicPartition tp = new TopicPartition(name, pi.partition()); + endOffsets.put(tp, admin.listOffsets(Collections.singletonMap(tp, OffsetSpec.latest())) + .all().get(timeout, TimeUnit.MILLISECONDS).get(tp)); + beginOffsets.put(tp, admin.listOffsets(Collections.singletonMap(tp, OffsetSpec.earliest())) + .all().get(timeout, TimeUnit.MILLISECONDS).get(tp)); + } + + long totalMessages = 0; + List> partitionStats = new ArrayList<>(); + for (TopicPartitionInfo pi : desc.partitions()) { + TopicPartition tp = new TopicPartition(name, pi.partition()); + long end = endOffsets.get(tp).offset(); + long begin = beginOffsets.get(tp).offset(); + long count = end - begin; + totalMessages += count; + + Map ps = new LinkedHashMap<>(); + ps.put("partition", pi.partition()); + ps.put("leader", pi.leader() != null ? pi.leader().id() : -1); + ps.put("replicas", pi.replicas().stream().map(Node::id).collect(Collectors.toList())); + ps.put("isr", pi.isr().stream().map(Node::id).collect(Collectors.toList())); + ps.put("beginOffset", begin); + ps.put("endOffset", end); + ps.put("messageCount", count); + partitionStats.add(ps); + } + + Map result = new LinkedHashMap<>(); + result.put("name", name); + result.put("partitions", desc.partitions().size()); + result.put("replicationFactor", desc.partitions().isEmpty() ? 0 + : desc.partitions().get(0).replicas().size()); + result.put("totalMessages", totalMessages); + result.put("partitionStats", partitionStats); + return result; + } + + private static Object getTopicConfig(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String name = stringOrEmpty(params, "name"); + + ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, name); + Config config = admin.describeConfigs(Collections.singletonList(resource)) + .all().get(timeout, TimeUnit.MILLISECONDS).get(resource); + + Map configs = new LinkedHashMap<>(); + for (ConfigEntry entry : config.entries()) { + Map entryMap = new LinkedHashMap<>(); + entryMap.put("value", entry.value()); + entryMap.put("source", entry.source().name()); + entryMap.put("isSensitive", entry.isSensitive()); + entryMap.put("isReadOnly", entry.isReadOnly()); + entryMap.put("isDefault", entry.isDefault()); + configs.put(entry.name(), entryMap); + } + return Collections.singletonMap("configs", configs); + } + + private static Object alterTopicConfig(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String name = stringOrEmpty(params, "name"); + + JsonArray entries = params.has("configs") && params.get("configs").isJsonArray() + ? params.getAsJsonArray("configs") : new JsonArray(); + + List ops = new ArrayList<>(); + for (JsonElement el : entries) { + JsonObject entry = el.getAsJsonObject(); + String key = entry.get("key").getAsString(); + String value = entry.has("value") && !entry.get("value").isJsonNull() + ? entry.get("value").getAsString() : null; + String opStr = stringOrDefault(entry, "op", "set"); + AlterConfigOp.OpType opType = switch (opStr.toLowerCase()) { + case "delete" -> AlterConfigOp.OpType.DELETE; + case "append" -> AlterConfigOp.OpType.APPEND; + case "subtract" -> AlterConfigOp.OpType.SUBTRACT; + default -> AlterConfigOp.OpType.SET; + }; + ops.add(new AlterConfigOp(new ConfigEntry(key, value), opType)); + } + + ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, name); + admin.incrementalAlterConfigs(Collections.singletonMap(resource, ops)) + .all().get(timeout, TimeUnit.MILLISECONDS); + return Collections.singletonMap("ok", true); + } + + // ----------------------------------------------------------------------- + // Consumer groups + // ----------------------------------------------------------------------- + + private static Object listConsumerGroups(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + Collection groups = admin.listConsumerGroups( + new ListConsumerGroupsOptions().timeoutMs(timeout)) + .all().get(timeout, TimeUnit.MILLISECONDS); + + List> result = new ArrayList<>(); + for (ConsumerGroupListing group : groups) { + Map g = new LinkedHashMap<>(); + g.put("groupId", group.groupId()); + g.put("state", group.state().map(Enum::name).orElse("UNKNOWN")); + g.put("simpleGroup", group.isSimpleConsumerGroup()); + result.add(g); + } + result.sort(Comparator.comparing(m -> (String) m.get("groupId"))); + return Collections.singletonMap("groups", result); + } + + private static Object describeConsumerGroup(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String groupId = stringOrEmpty(params, "groupId"); + + ConsumerGroupDescription desc = admin.describeConsumerGroups(Collections.singletonList(groupId)) + .all().get(timeout, TimeUnit.MILLISECONDS).get(groupId); + + Map result = new LinkedHashMap<>(); + result.put("groupId", desc.groupId()); + result.put("state", desc.state().name()); + result.put("coordinator", nodeToMap(desc.coordinator())); + result.put("partitionAssignor", desc.partitionAssignor()); + + List> members = new ArrayList<>(); + for (MemberDescription member : desc.members()) { + Map m = new LinkedHashMap<>(); + m.put("memberId", member.consumerId()); + m.put("clientId", member.clientId()); + m.put("host", member.host()); + List> assignments = new ArrayList<>(); + for (TopicPartition tp : member.assignment().topicPartitions()) { + Map a = new LinkedHashMap<>(); + a.put("topic", tp.topic()); + a.put("partition", tp.partition()); + assignments.add(a); + } + m.put("assignments", assignments); + members.add(m); + } + result.put("members", members); + return result; + } + + private static Object listProducers(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String topic = stringOrEmpty(params, "topic"); + + TopicDescription desc = admin.describeTopics(Collections.singletonList(topic)) + .allTopicNames().get(timeout, TimeUnit.MILLISECONDS).get(topic); + List partitions = desc.partitions().stream() + .map(pi -> new TopicPartition(topic, pi.partition())) + .collect(Collectors.toList()); + + DescribeProducersResult described = admin.describeProducers( + partitions, + new DescribeProducersOptions().timeoutMs(timeout)); + + Map> byProducer = new LinkedHashMap<>(); + for (TopicPartition tp : partitions) { + DescribeProducersResult.PartitionProducerState state = + described.partitionResult(tp).get(timeout, TimeUnit.MILLISECONDS); + for (ProducerState producerState : state.activeProducers()) { + long producerId = producerState.producerId(); + Map producer = byProducer.computeIfAbsent(producerId, id -> { + Map p = new LinkedHashMap<>(); + p.put("producerId", id); + p.put("producerName", "producer-" + id); + p.put("msgRateIn", 0.0); + p.put("msgThroughputIn", 0.0); + p.put("clientVersion", "Kafka producer"); + p.put("partitions", new ArrayList()); + p.put("lastTimestamp", producerState.lastTimestamp()); + return p; + }); + @SuppressWarnings("unchecked") + List producerPartitions = (List) producer.get("partitions"); + producerPartitions.add(tp.partition()); + long currentLastTimestamp = (long) producer.get("lastTimestamp"); + if (producerState.lastTimestamp() > currentLastTimestamp) { + producer.put("lastTimestamp", producerState.lastTimestamp()); + } + } + } + + for (Map producer : byProducer.values()) { + @SuppressWarnings("unchecked") + List producerPartitions = (List) producer.get("partitions"); + producerPartitions.sort(Integer::compareTo); + producer.put( + "address", + producerPartitions.size() == 1 + ? "partition " + producerPartitions.get(0) + : "partitions " + producerPartitions.stream().map(String::valueOf).collect(Collectors.joining(", ")) + ); + } + + return Collections.singletonMap("producers", new ArrayList<>(byProducer.values())); + } + + private static Object deleteConsumerGroup(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String groupId = stringOrEmpty(params, "groupId"); + admin.deleteConsumerGroups(Collections.singletonList(groupId)) + .all().get(timeout, TimeUnit.MILLISECONDS); + return Collections.singletonMap("ok", true); + } + + private static Object resetConsumerGroupOffsets(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String groupId = stringOrEmpty(params, "groupId"); + String topic = stringOrEmpty(params, "topic"); + + Map offsets = new HashMap<>(); + JsonArray offsetArray = params.has("offsets") && params.get("offsets").isJsonArray() + ? params.getAsJsonArray("offsets") : new JsonArray(); + + for (JsonElement el : offsetArray) { + JsonObject offsetObj = el.getAsJsonObject(); + int partition = offsetObj.get("partition").getAsInt(); + long offset = offsetObj.get("offset").getAsLong(); + offsets.put(new TopicPartition(topic, partition), new OffsetAndMetadata(offset)); + } + + // If no explicit offsets, check for a "position" parameter. + if (offsets.isEmpty()) { + String position = stringOrDefault(params, "position", "latest"); + Long timestampMs = params.has("timestampMs") && !params.get("timestampMs").isJsonNull() + ? params.get("timestampMs").getAsLong() : null; + TopicDescription desc = admin.describeTopics(Collections.singletonList(topic)) + .allTopicNames().get(timeout, TimeUnit.MILLISECONDS).get(topic); + + Map specMap = new HashMap<>(); + for (TopicPartitionInfo pi : desc.partitions()) { + TopicPartition tp = new TopicPartition(topic, pi.partition()); + specMap.put(tp, offsetSpecForPosition(position, timestampMs)); + } + Map resolved = + admin.listOffsets(specMap).all().get(timeout, TimeUnit.MILLISECONDS); + List unresolvedTimestampPartitions = new ArrayList<>(); + for (Map.Entry entry : resolved.entrySet()) { + long offset = entry.getValue().offset(); + if (offset >= 0) { + offsets.put(entry.getKey(), new OffsetAndMetadata(offset)); + } else { + unresolvedTimestampPartitions.add(entry.getKey()); + } + } + if (!unresolvedTimestampPartitions.isEmpty()) { + Map latestSpecs = new HashMap<>(); + for (TopicPartition tp : unresolvedTimestampPartitions) { + latestSpecs.put(tp, OffsetSpec.latest()); + } + Map latest = + admin.listOffsets(latestSpecs).all().get(timeout, TimeUnit.MILLISECONDS); + for (Map.Entry entry : latest.entrySet()) { + offsets.put(entry.getKey(), new OffsetAndMetadata(entry.getValue().offset())); + } + } + } + + admin.alterConsumerGroupOffsets(groupId, offsets) + .all().get(timeout, TimeUnit.MILLISECONDS); + return Collections.singletonMap("ok", true); + } + + static OffsetSpec offsetSpecForPosition(String position, Long timestampMs) { + String normalized = position == null ? "latest" : position.trim().toLowerCase(Locale.ROOT); + return switch (normalized) { + case "earliest" -> OffsetSpec.earliest(); + case "latest", "" -> OffsetSpec.latest(); + case "timestamp" -> { + if (timestampMs == null) { + throw new IllegalArgumentException("timestampMs is required when position is timestamp"); + } + yield OffsetSpec.forTimestamp(timestampMs); + } + default -> throw new IllegalArgumentException("Unsupported reset position: " + position); + }; + } + + // ----------------------------------------------------------------------- + // Messages + // ----------------------------------------------------------------------- + + private static Object peekMessages(JsonObject params) throws Exception { + String topic = stringOrEmpty(params, "topic"); + int partition = intOrDefault(params, "partition", 0); + long offset = longOrDefault(params, "offset", 0); + int count = intOrDefault(params, "count", 10); + + // Build a temporary consumer for peeking (no commit) + Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + adminClient != null ? adminClient.describeCluster().clusterId() + .get(5, TimeUnit.SECONDS) : "localhost:9092"); + // Reuse the admin's bootstrap servers + JsonObject conn = params.has("connection") && params.get("connection").isJsonObject() + ? params.getAsJsonObject("connection") : null; + if (conn != null) { + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn)); + applyConnectionProperties(conn, props); + } + props.put(ConsumerConfig.GROUP_ID_CONFIG, "dbx-peek-" + UUID.randomUUID()); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringDeserializer"); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none"); + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, count); + + TopicPartition tp = new TopicPartition(topic, partition); + try (KafkaConsumer consumer = new KafkaConsumer<>(props)) { + consumer.assign(Collections.singletonList(tp)); + consumer.seek(tp, offset); + + List> messages = new ArrayList<>(); + ConsumerRecords records = consumer.poll(Duration.ofSeconds(5)); + for (ConsumerRecord record : records) { + if (messages.size() >= count) break; + Map msg = new LinkedHashMap<>(); + msg.put("topic", record.topic()); + msg.put("partition", record.partition()); + msg.put("offset", record.offset()); + msg.put("timestamp", record.timestamp()); + msg.put("key", record.key()); + // Headers + Map headers = new LinkedHashMap<>(); + record.headers().forEach(h -> + headers.put(h.key(), new String(h.value(), StandardCharsets.UTF_8))); + msg.put("headers", headers); + // Payload + if (record.value() != null) { + msg.put("payloadBase64", Base64.getEncoder().encodeToString(record.value())); + String text = tryDecodeUtf8(record.value()); + if (text != null) { + msg.put("payloadText", text); + } + } else { + msg.put("payloadBase64", ""); + } + messages.add(msg); + } + return Collections.singletonMap("messages", messages); + } + } + + private static Object sendMessage(JsonObject params) throws Exception { + if (producer == null) { + throw new IllegalStateException("Producer is not initialized. Call connect first."); + } + + String topic = stringOrEmpty(params, "topic"); + String key = params.has("key") && !params.get("key").isJsonNull() + ? params.get("key").getAsString() : null; + + // Decode payload from base64 + String payloadBase64 = stringOrEmpty(params, "payloadBase64"); + byte[] value = payloadBase64.isEmpty() ? new byte[0] : Base64.getDecoder().decode(payloadBase64); + + // Build the record + Integer partition = params.has("partition") && !params.get("partition").isJsonNull() + ? params.get("partition").getAsInt() : null; + + ProducerRecord record; + if (partition != null) { + record = new ProducerRecord<>(topic, partition, key, value); + } else { + record = new ProducerRecord<>(topic, key, value); + } + + // Add headers + JsonObject headers = params.has("headers") && params.get("headers").isJsonObject() + ? params.getAsJsonObject("headers") : null; + if (headers != null) { + for (Map.Entry entry : headers.entrySet()) { + record.headers().add(new RecordHeader( + entry.getKey(), + entry.getValue().getAsString().getBytes(StandardCharsets.UTF_8))); + } + } + + RecordMetadata metadata = producer.send(record).get(30, TimeUnit.SECONDS); + + Map result = new LinkedHashMap<>(); + result.put("ok", true); + result.put("topic", metadata.topic()); + result.put("partition", metadata.partition()); + result.put("offset", metadata.offset()); + result.put("timestamp", metadata.timestamp()); + return result; + } + + // ----------------------------------------------------------------------- + // ACLs + // ----------------------------------------------------------------------- + + private static Object listAcls(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + + AclBindingFilter filter = buildAclFilter(params); + Collection bindings = admin.describeAcls(filter) + .values().get(timeout, TimeUnit.MILLISECONDS); + + List> acls = new ArrayList<>(); + for (AclBinding binding : bindings) { + Map acl = new LinkedHashMap<>(); + acl.put("resourceType", binding.pattern().resourceType().name()); + acl.put("resourceName", binding.pattern().name()); + acl.put("patternType", binding.pattern().patternType().name()); + acl.put("principal", binding.entry().principal()); + acl.put("host", binding.entry().host()); + acl.put("operation", binding.entry().operation().name()); + acl.put("permissionType", binding.entry().permissionType().name()); + acls.add(acl); + } + return Collections.singletonMap("acls", acls); + } + + private static Object createAcls(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + JsonArray aclsArray = params.has("acls") && params.get("acls").isJsonArray() + ? params.getAsJsonArray("acls") : new JsonArray(); + + List bindings = new ArrayList<>(); + for (JsonElement el : aclsArray) { + JsonObject acl = el.getAsJsonObject(); + ResourceType resourceType = ResourceType.valueOf(stringOrDefault(acl, "resourceType", "TOPIC")); + String resourceName = stringOrEmpty(acl, "resourceName"); + PatternType patternType = PatternType.valueOf(stringOrDefault(acl, "patternType", "LITERAL")); + String principal = stringOrEmpty(acl, "principal"); + String host = stringOrDefault(acl, "host", "*"); + AclOperation operation = AclOperation.valueOf(stringOrDefault(acl, "operation", "ALL")); + AclPermissionType permissionType = AclPermissionType.valueOf( + stringOrDefault(acl, "permissionType", "ALLOW")); + + ResourcePattern pattern = new ResourcePattern(resourceType, resourceName, patternType); + AccessControlEntry entry = new AccessControlEntry(principal, host, operation, permissionType); + bindings.add(new AclBinding(pattern, entry)); + } + + admin.createAcls(bindings).all().get(timeout, TimeUnit.MILLISECONDS); + return Collections.singletonMap("ok", true); + } + + private static Object deleteAcls(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + JsonArray filtersArray = params.has("filters") && params.get("filters").isJsonArray() + ? params.getAsJsonArray("filters") : new JsonArray(); + + List filters = new ArrayList<>(); + for (JsonElement el : filtersArray) { + filters.add(buildAclFilter(el.getAsJsonObject())); + } + if (filters.isEmpty()) { + filters.add(AclBindingFilter.ANY); + } + + Collection deletedBindings = admin.deleteAcls(filters).all().get(timeout, TimeUnit.MILLISECONDS); + int deleted = deletedAclCount(deletedBindings); + Map result = new LinkedHashMap<>(); + result.put("ok", true); + result.put("deleted", deleted); + return result; + } + + static int deletedAclCount(Collection deletedBindings) { + return deletedBindings == null ? 0 : deletedBindings.size(); + } + + private static AclBindingFilter buildAclFilter(JsonObject params) { + ResourceType resourceType = params.has("resourceType") + ? ResourceType.valueOf(params.get("resourceType").getAsString()) : ResourceType.ANY; + String resourceName = params.has("resourceName") && !params.get("resourceName").isJsonNull() + ? params.get("resourceName").getAsString() : null; + PatternType patternType = params.has("patternType") + ? PatternType.valueOf(params.get("patternType").getAsString()) : PatternType.ANY; + + ResourcePatternFilter patternFilter = new ResourcePatternFilter( + resourceType, resourceName, patternType); + + String principal = params.has("principal") && !params.get("principal").isJsonNull() + ? params.get("principal").getAsString() : null; + String host = params.has("host") && !params.get("host").isJsonNull() + ? params.get("host").getAsString() : null; + AclOperation operation = params.has("operation") + ? AclOperation.valueOf(params.get("operation").getAsString()) : AclOperation.ANY; + AclPermissionType permissionType = params.has("permissionType") + ? AclPermissionType.valueOf(params.get("permissionType").getAsString()) : AclPermissionType.ANY; + + AccessControlEntryFilter entryFilter = new AccessControlEntryFilter( + principal, host, operation, permissionType); + return new AclBindingFilter(patternFilter, entryFilter); + } + + // ----------------------------------------------------------------------- + // Cluster / monitoring + // ----------------------------------------------------------------------- + + private static Object describeCluster(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + + DescribeClusterResult cluster = admin.describeCluster(); + String clusterId = cluster.clusterId().get(timeout, TimeUnit.MILLISECONDS); + Node controller = cluster.controller().get(timeout, TimeUnit.MILLISECONDS); + Collection nodes = cluster.nodes().get(timeout, TimeUnit.MILLISECONDS); + + Map result = new LinkedHashMap<>(); + result.put("clusterId", clusterId); + result.put("controller", controller != null ? nodeToMap(controller) : null); + List> brokerList = new ArrayList<>(); + for (Node node : nodes) { + brokerList.add(nodeToMap(node)); + } + result.put("brokers", brokerList); + result.put("nodeCount", nodes.size()); + return result; + } + + private static Object getConsumerLag(JsonObject params) throws Exception { + AdminClient admin = requireAdmin(); + int timeout = requestTimeout(params); + String groupId = stringOrEmpty(params, "groupId"); + String topic = stringOrEmpty(params, "topic"); + + // Get committed offsets for the consumer group + Map committed = admin.listConsumerGroupOffsets(groupId) + .partitionsToOffsetAndMetadata().get(timeout, TimeUnit.MILLISECONDS); + + // Filter to the requested topic + Map topicCommitted = committed.entrySet().stream() + .filter(e -> e.getKey().topic().equals(topic)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + if (topicCommitted.isEmpty()) { + return Collections.singletonMap("partitions", Collections.emptyList()); + } + + // Get end offsets + Map specMap = new HashMap<>(); + for (TopicPartition tp : topicCommitted.keySet()) { + specMap.put(tp, OffsetSpec.latest()); + } + Map endOffsets = + admin.listOffsets(specMap).all().get(timeout, TimeUnit.MILLISECONDS); + + List> partitions = new ArrayList<>(); + long totalLag = 0; + for (Map.Entry entry : topicCommitted.entrySet()) { + TopicPartition tp = entry.getKey(); + long currentOffset = entry.getValue().offset(); + ListOffsetsResult.ListOffsetsResultInfo endInfo = endOffsets.get(tp); + long endOffset = endInfo != null ? endInfo.offset() : -1; + long lag = endOffset >= 0 ? Math.max(0, endOffset - currentOffset) : -1; + totalLag += Math.max(0, lag); + + Map p = new LinkedHashMap<>(); + p.put("partition", tp.partition()); + p.put("currentOffset", currentOffset); + p.put("endOffset", endOffset); + p.put("lag", lag); + partitions.add(p); + } + partitions.sort(Comparator.comparingInt(a -> (int) a.get("partition"))); + + Map result = new LinkedHashMap<>(); + result.put("partitions", partitions); + result.put("totalLag", totalLag); + return result; + } + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + private static AdminClient requireAdmin() { + if (adminClient == null) { + throw new IllegalStateException("Not connected. Call connect first."); + } + return adminClient; + } + + private static JsonObject connectionObject(JsonObject params) { + JsonElement connection = params.get("connection"); + return connection != null && connection.isJsonObject() + ? connection.getAsJsonObject() : params; + } + + private static int requestTimeout(JsonObject params) { + return intOrDefault(params, "timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS); + } + + private static Map nodeToMap(Node node) { + Map m = new LinkedHashMap<>(); + m.put("id", node.id()); + m.put("host", node.host()); + m.put("port", node.port()); + m.put("rack", node.rack()); + return m; + } + + private static String tryDecodeUtf8(byte[] bytes) { + try { + String text = new String(bytes, StandardCharsets.UTF_8); + // Verify round-trip + byte[] reEncoded = text.getBytes(StandardCharsets.UTF_8); + if (Arrays.equals(bytes, reEncoded)) { + return text; + } + } catch (Exception ignored) {} + return null; + } + + private static String stringOrNull(JsonObject object, String key) { + JsonElement element = object.get(key); + return element == null || element.isJsonNull() ? null : element.getAsString(); + } + + private static String stringOrEmpty(JsonObject object, String key) { + return stringOrDefault(object, key, ""); + } + + private static String stringOrDefault(JsonObject object, String key, String fallback) { + String value = stringOrNull(object, key); + return value == null ? fallback : value; + } + + private static int intOrDefault(JsonObject object, String key, int fallback) { + JsonElement element = object.get(key); + return element == null || element.isJsonNull() ? fallback : element.getAsInt(); + } + + private static long longOrDefault(JsonObject object, String key, long fallback) { + JsonElement element = object.get(key); + return element == null || element.isJsonNull() ? fallback : element.getAsLong(); + } + + private static boolean boolOrDefault(JsonObject object, String key, boolean fallback) { + JsonElement element = object.get(key); + return element == null || element.isJsonNull() ? fallback : element.getAsBoolean(); + } + + // ----------------------------------------------------------------------- + // Inner types + // ----------------------------------------------------------------------- + + private static final class HandshakeResult { + private final int protocolVersion; + private final int agentProtocolVersion; + private final List capabilities; + + private HandshakeResult(int protocolVersion, int agentProtocolVersion, List capabilities) { + this.protocolVersion = protocolVersion; + this.agentProtocolVersion = agentProtocolVersion; + this.capabilities = capabilities; + } + } +} diff --git a/agents/scripts/validate_agents.py b/agents/scripts/validate_agents.py index db771cbe2..f3f79568e 100644 --- a/agents/scripts/validate_agents.py +++ b/agents/scripts/validate_agents.py @@ -11,7 +11,7 @@ SOURCE_GLOBS = ("*/src/main/**/*.java", "drivers/*/src/main/**/*.java") KOTLIN_FILE_SUFFIXES = (".kt", ".kts") KOTLIN_SCAN_EXCLUDED_PARTS = {".git", ".gradle", "build"} DEFAULT_AGENT_JRE_KEY = "21" -NON_JDBC_AGENT_MODULES = {"mongodb", "etcd", "zookeeper"} +NON_JDBC_AGENT_MODULES = {"mongodb", "etcd", "zookeeper", "kafka"} NATIVE_ONLY_AGENT_MODULES = { "oracle": "drivers/oracle-go", "xugu": "drivers/xugu", diff --git a/agents/settings.gradle b/agents/settings.gradle index 5f3ada0ff..38a228596 100644 --- a/agents/settings.gradle +++ b/agents/settings.gradle @@ -6,7 +6,7 @@ def driverModules = [ 'teradata', 'vertica', 'firebird', 'exasol', 'oceanbase-oracle', 'gbase8a', 'gbase8s', 'bigquery', 'kylin', 'sundb', 'h2', 'snowflake', 'trino', 'hive', 'db2', 'informix', 'neo4j', 'cassandra', 'mongodb', 'highgo', 'tdengine', 'yashandb', - 'iris', 'iotdb', 'etcd', 'zookeeper' + 'iris', 'iotdb', 'etcd', 'zookeeper', 'kafka' ] include(*(infrastructureModules + driverModules)) diff --git a/agents/versions.json b/agents/versions.json index d6a359b39..b4b9b2b7e 100644 --- a/agents/versions.json +++ b/agents/versions.json @@ -34,5 +34,6 @@ "iris": "0.1.16", "iotdb": "0.1.13", "etcd": "0.1.13", - "zookeeper": "0.1.3" + "zookeeper": "0.1.3", + "kafka": "0.1.0" } diff --git a/apps/desktop/public/icons/database/kafka.svg b/apps/desktop/public/icons/database/kafka.svg new file mode 100644 index 000000000..cb90e601d --- /dev/null +++ b/apps/desktop/public/icons/database/kafka.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/desktop/src/components/connection/ConnectionDialog.vue b/apps/desktop/src/components/connection/ConnectionDialog.vue index 4ef64cd32..b9c65b247 100644 --- a/apps/desktop/src/components/connection/ConnectionDialog.vue +++ b/apps/desktop/src/components/connection/ConnectionDialog.vue @@ -321,8 +321,12 @@ const dbPickerView = ref("icon"); const dbSearchQuery = ref(""); const configTab = ref("connection"); type MqAuthKind = MqAuth["kind"]; +const MQ_KAFKA_SECURITY_PROTOCOL_AUTO = "__auto"; const mqAdminUrl = ref("http://127.0.0.1:8080"); const mqSystemKind = ref("pulsar"); +const mqKafkaBootstrapServers = ref("127.0.0.1:9092"); +const mqKafkaSecurityProtocol = ref(MQ_KAFKA_SECURITY_PROTOCOL_AUTO); +const mqKafkaSaslMechanism = ref("PLAIN"); const mqAuthKind = ref("none"); const mqToken = ref(""); const mqBasicUsername = ref(""); @@ -338,6 +342,22 @@ const mqTlsSkipVerify = ref(false); const mqPinnedVersion = ref(pinnedVersionToSelection(undefined)); const mqTokenSigningMode = ref("none"); const mqTokenSigningKey = ref(""); +const mqSystemOptions: Array<{ value: MqSystemKind; label: string }> = [ + { value: "pulsar", label: "Apache Pulsar" }, + { value: "kafka", label: "Apache Kafka" }, +]; +const mqKafkaSecurityProtocolOptions = [ + { value: MQ_KAFKA_SECURITY_PROTOCOL_AUTO, label: "Auto" }, + { value: "PLAINTEXT", label: "PLAINTEXT" }, + { value: "SSL", label: "SSL" }, + { value: "SASL_PLAINTEXT", label: "SASL_PLAINTEXT" }, + { value: "SASL_SSL", label: "SASL_SSL" }, +]; +const mqKafkaSaslMechanismOptions = [ + { value: "PLAIN", label: "PLAIN" }, + { value: "SCRAM-SHA-256", label: "SCRAM-SHA-256" }, + { value: "SCRAM-SHA-512", label: "SCRAM-SHA-512" }, +]; const nacosServerAddr = ref(NACOS_DEFAULT_CONSOLE_URL); const nacosNamespace = ref(""); const nacosContextPath = ref(""); @@ -552,6 +572,7 @@ const driverProfiles: Record< etcd: { type: "etcd", port: 2379, user: "", label: "etcd", icon: "etcd" }, zookeeper: { type: "zookeeper", port: 2181, user: "", label: "Apache ZooKeeper", icon: "zookeeper" }, mq: { type: "mq", port: 8080, user: "", label: "Apache Pulsar", icon: "pulsar", host: "127.0.0.1" }, + kafka: { type: "mq", port: 9092, user: "", label: "Apache Kafka", icon: "kafka", host: "127.0.0.1" }, nacos: { type: "nacos", port: 8848, user: "nacos", label: "Nacos", icon: "nacos", host: "127.0.0.1" }, iris: { type: "iris", port: 1972, user: "_SYSTEM", label: "IRIS", icon: "iris" }, influxdb: { type: "influxdb", port: 8086, user: "", label: "InfluxDB", icon: "InfluxDB" }, @@ -579,6 +600,8 @@ function profileForConfig(config: ConnectionConfig) { if (config.driver_profile === "oceanbase-oracle") return "oceanbase"; return config.driver_profile; } + if (config.db_type === "mq" && (config.external_config as MqAdminConfig | undefined)?.systemKind === "kafka") return "kafka"; + if (config.db_type === "mq") return "mq"; if (config.db_type === "dameng") return "dm"; if (config.db_type === "oceanbase-oracle") return "oceanbase"; return config.db_type; @@ -588,13 +611,28 @@ function selectedProfile() { return driverProfiles[selectedType.value] ?? driverProfiles.mysql; } +function mqExtraRecord(config?: Partial): Record { + const extra = config?.extra; + return extra && typeof extra === "object" && !Array.isArray(extra) ? (extra as Record) : {}; +} + +function mqExtraString(extra: Record, key: string): string { + const value = extra[key]; + return typeof value === "string" ? value : ""; +} + function resetMqFields(config?: Partial) { - mqSystemKind.value = "pulsar"; - mqAdminUrl.value = config?.adminUrl?.trim() || "http://127.0.0.1:8080"; + const systemKind = config?.systemKind === "kafka" ? "kafka" : "pulsar"; + const extra = mqExtraRecord(config); + mqSystemKind.value = systemKind; + mqAdminUrl.value = config?.adminUrl?.trim() || (systemKind === "kafka" ? "" : "http://127.0.0.1:8080"); + mqKafkaBootstrapServers.value = mqExtraString(extra, "bootstrapServers") || "127.0.0.1:9092"; + mqKafkaSecurityProtocol.value = mqExtraString(extra, "securityProtocol") || MQ_KAFKA_SECURITY_PROTOCOL_AUTO; + mqKafkaSaslMechanism.value = mqExtraString(extra, "saslMechanism") || "PLAIN"; mqTlsSkipVerify.value = !!config?.tlsSkipVerify; mqPinnedVersion.value = pinnedVersionToSelection(config?.pinnedVersion); const auth = (config?.auth || { kind: "none" }) as MqAuth; - mqAuthKind.value = auth.kind || "none"; + mqAuthKind.value = systemKind === "kafka" && auth.kind !== "basic" ? "none" : auth.kind || "none"; mqToken.value = auth.token || ""; mqBasicUsername.value = auth.username || ""; mqBasicPassword.value = auth.password || ""; @@ -610,6 +648,16 @@ function resetMqFields(config?: Partial) { mqTokenSigningKey.value = tokenSigning?.key || ""; } +function defaultMqFieldsForProfile(profile: string): Partial | undefined { + if (profile !== "kafka") return undefined; + return { + systemKind: "kafka", + adminUrl: "", + auth: { kind: "none" }, + extra: { bootstrapServers: "127.0.0.1:9092" }, + }; +} + function hydrateMqFields(value: unknown) { if (!value || typeof value !== "object") { resetMqFields(); @@ -618,6 +666,15 @@ function hydrateMqFields(value: unknown) { resetMqFields(value as Partial); } +watch(mqSystemKind, (kind) => { + if (kind === "kafka") { + if (!mqKafkaBootstrapServers.value.trim()) mqKafkaBootstrapServers.value = "127.0.0.1:9092"; + if (!["none", "basic"].includes(mqAuthKind.value)) mqAuthKind.value = "none"; + return; + } + if (!mqAdminUrl.value.trim()) mqAdminUrl.value = "http://127.0.0.1:8080"; +}); + function resetNacosFields(config?: Partial) { nacosServerAddr.value = config?.serverAddr?.trim() || NACOS_DEFAULT_CONSOLE_URL; nacosNamespace.value = config?.namespace || ""; @@ -644,6 +701,32 @@ function requireMqField(value: string, message: string): string { return trimmed; } +function normalizeMqKafkaBootstrapServer(server: string): string { + if (server.includes("://")) { + throw new Error("Kafka bootstrap servers must be host:port values without a URL scheme"); + } + let parsed: URL; + try { + parsed = new URL(`kafka://${server}`); + } catch { + throw new Error("Kafka bootstrap servers are invalid"); + } + if (!parsed.hostname || parsed.username || parsed.password || parsed.search || parsed.hash || (parsed.pathname && parsed.pathname !== "/")) { + throw new Error("Kafka bootstrap servers are invalid"); + } + return server; +} + +function normalizeMqKafkaBootstrapServers(value: string): string { + const servers = requireMqField(value, "Kafka bootstrap servers are required") + .split(",") + .map((server) => server.trim()) + .filter(Boolean) + .map(normalizeMqKafkaBootstrapServer); + if (!servers.length) throw new Error("Kafka bootstrap servers are required"); + return servers.join(","); +} + function buildMqAuth(): MqAuth { switch (mqAuthKind.value) { case "token": @@ -683,8 +766,25 @@ function buildMqTokenSigning() { } function buildMqAdminConfig(): MqAdminConfig { + const systemKind = mqSystemKind.value; + if (systemKind === "kafka") { + const bootstrapServers = normalizeMqKafkaBootstrapServers(mqKafkaBootstrapServers.value); + const extra: Record = { bootstrapServers }; + const securityProtocol = mqKafkaSecurityProtocol.value === MQ_KAFKA_SECURITY_PROTOCOL_AUTO ? "" : mqKafkaSecurityProtocol.value.trim(); + const saslMechanism = mqKafkaSaslMechanism.value.trim(); + if (securityProtocol) extra.securityProtocol = securityProtocol; + if (mqAuthKind.value === "basic" && saslMechanism) extra.saslMechanism = saslMechanism; + return { + systemKind: mqSystemKind.value, + adminUrl: "", + auth: buildMqAuth(), + tlsSkipVerify: mqTlsSkipVerify.value || undefined, + extra, + }; + } + return { - systemKind: "pulsar", + systemKind: mqSystemKind.value, adminUrl: requireMqField(mqAdminUrl.value, "MQ Admin URL is required"), auth: buildMqAuth(), tlsSkipVerify: mqTlsSkipVerify.value || undefined, @@ -804,6 +904,20 @@ function applyMqAdminUrl(config: LegacyConnectionConfig, adminUrl: string) { config.ssl = parsed.protocol === "https:"; } +function applyMqKafkaBootstrapServers(config: LegacyConnectionConfig, bootstrapServers: string, securityProtocol?: string) { + const first = normalizeMqKafkaBootstrapServers(bootstrapServers).split(",")[0]; + if (!first) throw new Error("Kafka bootstrap servers are required"); + let parsed: URL; + try { + parsed = new URL(`kafka://${first}`); + } catch { + throw new Error("Kafka bootstrap servers are invalid"); + } + config.host = parsed.hostname; + config.port = Number(parsed.port) || 9092; + config.ssl = securityProtocol === "SSL" || securityProtocol === "SASL_SSL"; +} + function applyNacosServerAddr(config: LegacyConnectionConfig, serverAddr: string) { let parsed: URL; try { @@ -934,7 +1048,7 @@ function applyProfile(val: string, preserveConnectionFields = false) { applyPrestoSqlBuiltinDriverPathsIfAvailable(); } if (profile.type === "mq") { - resetMqFields(); + resetMqFields(defaultMqFieldsForProfile(val)); form.value.database = undefined; form.value.connection_string = undefined; } @@ -1202,6 +1316,7 @@ const iconTypeMap: Record = { etcd: "etcd", zookeeper: "zookeeper", mq: "mq", + kafka: "kafka", nacos: "nacos", dm: "dm", h2: "h2", @@ -1289,6 +1404,7 @@ const dbOptions: DbOption[] = [ { value: "etcd", label: "etcd" }, { value: "zookeeper", label: "Apache ZooKeeper" }, { value: "mq", label: "Apache Pulsar" }, + { value: "kafka", label: "Apache Kafka" }, { value: "nacos", label: "Nacos" }, { value: "influxdb", label: "InfluxDB" }, { value: "iris", label: "IRIS" }, @@ -1497,7 +1613,9 @@ const connectionLabelSmallClass = `${connectionLabelClass} text-xs`; const connectionLabelTopClass = `${connectionLabelClass} mt-2`; const connectionLabelSmallPaddedClass = `${connectionLabelClass} pt-2 text-xs`; const hasRequiredConnectionTarget = computed(() => { - if (form.value.db_type === "mq") return !!mqAdminUrl.value.trim(); + if (form.value.db_type === "mq") { + return mqSystemKind.value === "kafka" ? !!mqKafkaBootstrapServers.value.trim() : !!mqAdminUrl.value.trim(); + } if (form.value.db_type === "zookeeper") return !!(form.value.host || form.value.connection_string || connectionUrlInput.value.trim()); if (form.value.db_type === "nacos") return !!nacosServerAddr.value.trim(); if (isH2FileMode.value) return !!(form.value.host.trim() || h2FilePathFromJdbcUrl(form.value.connection_string)); @@ -1745,7 +1863,14 @@ function connectionConfigForSubmit(id: string): ConnectionConfig { if (config.db_type === "mq") { const mqConfig = buildMqAdminConfig(); config.external_config = mqConfig; - applyMqAdminUrl(config, mqConfig.adminUrl); + config.driver_profile = mqConfig.systemKind === "kafka" ? "kafka" : "pulsar"; + config.driver_label = mqConfig.systemKind === "kafka" ? "Apache Kafka" : "Apache Pulsar"; + if (mqConfig.systemKind === "kafka") { + const extra = mqExtraRecord(mqConfig); + applyMqKafkaBootstrapServers(config, mqExtraString(extra, "bootstrapServers"), mqExtraString(extra, "securityProtocol")); + } else { + applyMqAdminUrl(config, mqConfig.adminUrl); + } config.username = ""; config.password = ""; config.database = undefined; @@ -3265,22 +3390,52 @@ function openExternalUrl(url: string) {