diff --git a/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java b/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java index 5236ba7f9..b64429950 100644 --- a/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java +++ b/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java @@ -4,6 +4,7 @@ import com.google.gson.*; import org.apache.rocketmq.acl.common.AclClientRPCHook; import org.apache.rocketmq.acl.common.SessionCredentials; import org.apache.rocketmq.client.QueryResult; +import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; @@ -20,11 +21,13 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; import org.apache.rocketmq.remoting.protocol.body.AclInfo; import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; import org.apache.rocketmq.remoting.protocol.body.Connection; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; +import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.ProducerConnection; import org.apache.rocketmq.remoting.protocol.body.ProducerInfo; @@ -61,6 +64,11 @@ public final class RocketMqAgent { private static final int DEFAULT_LIST_LIMIT = 200; private static final int CONSUMER_GROUP_ENRICH_CONCURRENCY = 8; private static final long CONSUMER_GROUP_ENRICH_BUDGET_MS = 8_000; + /** Wall-clock budget for parallel getAllSubscriptionGroup across masters. */ + private static final long CONSUMER_GROUP_COLLECT_BUDGET_MS = 12_000; + private static final int CONSUMER_GROUP_COLLECT_CONCURRENCY = 8; + private static final long CONSUMER_LAG_BUDGET_MS = 10_000; + private static final int CONSUMER_LAG_CONCURRENCY = 8; /** Nameserver route enrichment for listTopics fallback when bulk broker config is unavailable. */ private static final int TOPIC_LIST_ROUTE_CONCURRENCY = 8; private static final long TOPIC_LIST_ROUTE_BUDGET_MS = 8_000; @@ -93,12 +101,17 @@ public final class RocketMqAgent { /** * Align with rocketmq-dashboard ConsumerServiceImpl group type: SYSTEM / FIFO / NORMAL. + * When the subscription dump is missing for a discovered group, return UNKNOWN instead of + * silently labeling FIFO groups as NORMAL. */ static String classifyConsumerGroupType(String groupId, SubscriptionGroupConfig config) { if (isSystemConsumerGroup(groupId)) { return "SYSTEM"; } - if (config != null && config.isConsumeMessageOrderly()) { + if (config == null) { + return "UNKNOWN"; + } + if (config.isConsumeMessageOrderly()) { return "FIFO"; } return "NORMAL"; @@ -1090,38 +1103,150 @@ public final class RocketMqAgent { private static SubscriptionGroupConfig findSubscriptionGroupConfig( DefaultMQAdminExt admin, JsonObject conn, String groupId) throws Exception { + // Walk every master and FIFO-merge so an earlier NORMAL stub cannot hide orderly=true. + long timeoutMs = Math.max(1_000L, intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS)); + Map merged = new HashMap<>(); for (String brokerAddr : resolveMasterBrokerAddrs(admin, conn)) { try { - SubscriptionGroupWrapper wrapper = admin.getAllSubscriptionGroup(brokerAddr, DEFAULT_REQUEST_TIMEOUT_MS); + SubscriptionGroupWrapper wrapper = admin.getAllSubscriptionGroup(brokerAddr, timeoutMs); if (wrapper == null || wrapper.getSubscriptionGroupTable() == null) { continue; } SubscriptionGroupConfig config = wrapper.getSubscriptionGroupTable().get(groupId); if (config != null) { - return config; + mergeSubscriptionGroupConfigs(merged, Map.of(groupId, config)); } } catch (Exception ignored) { // Try next broker. } } - return null; + return merged.get(groupId); } private static Map collectConsumerGroupConfigs( DefaultMQAdminExt admin, JsonObject conn) throws Exception { + long requestTimeoutMs = intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS); + // Prefer connection RPC timeout over the legacy 5s hard-code so slow-but-alive brokers + // still contribute within the shared wall-clock budget. + long perBrokerMs = Math.min(Math.max(1_000L, requestTimeoutMs), CONSUMER_GROUP_COLLECT_BUDGET_MS); + return collectConsumerGroupConfigs( + admin, + conn, + CONSUMER_GROUP_COLLECT_CONCURRENCY, + CONSUMER_GROUP_COLLECT_BUDGET_MS, + perBrokerMs + ); + } + + /** + * Topic-filtered lists skip the parallel full-cluster collect for speed, but must still + * walk every reachable master. An intermediate broker can return a duplicate dump (map size + * unchanged) while a later master still holds FIFO-only groups. + */ + private static Map collectConsumerGroupConfigsForTopicFilter( + DefaultMQAdminExt admin, JsonObject conn) { + long timeoutMs = Math.max(1_000L, intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS)); Map configs = new TreeMap<>(); - for (String brokerAddr : resolveMasterBrokerAddrs(admin, conn)) { - try { - SubscriptionGroupWrapper wrapper = admin.getAllSubscriptionGroup(brokerAddr, DEFAULT_REQUEST_TIMEOUT_MS); - if (wrapper == null || wrapper.getSubscriptionGroupTable() == null) { + try { + for (String brokerAddr : resolveMasterBrokerAddrs(admin, conn)) { + try { + SubscriptionGroupWrapper wrapper = admin.getAllSubscriptionGroup(brokerAddr, timeoutMs); + if (wrapper == null || wrapper.getSubscriptionGroupTable() == null + || wrapper.getSubscriptionGroupTable().isEmpty()) { + continue; + } + mergeSubscriptionGroupConfigs(configs, wrapper.getSubscriptionGroupTable()); + } catch (Exception ignored) { + // Try next master. + } + } + } catch (Exception ignored) { + // No reachable masters. + } + return configs; + } + + /** + * Merge broker subscription dumps. Prefer {@code consumeMessageOrderly=true} so a NORMAL + * stub on an earlier master cannot hide a FIFO group that exists on another broker. + */ + static void mergeSubscriptionGroupConfigs( + Map target, + Map incoming + ) { + if (incoming == null || incoming.isEmpty()) { + return; + } + for (Map.Entry entry : incoming.entrySet()) { + String groupId = entry.getKey(); + SubscriptionGroupConfig next = entry.getValue(); + if (groupId == null || next == null) { + continue; + } + target.merge(groupId, next, (existing, candidate) -> { + if (candidate != null && candidate.isConsumeMessageOrderly() + && (existing == null || !existing.isConsumeMessageOrderly())) { + return candidate; + } + return existing != null ? existing : candidate; + }); + } + } + + /** + * Parallel subscription-group dump with a total budget. Slow/unreachable brokers are + * skipped so a large cluster does not burn the entire Agent RPC window. + */ + static Map collectConsumerGroupConfigs( + DefaultMQAdminExt admin, + JsonObject conn, + int concurrency, + long budgetMs, + long perBrokerTimeoutMs + ) throws Exception { + List brokers = resolveMasterBrokerAddrs(admin, conn); + Map configs = new TreeMap<>(); + if (brokers.isEmpty()) { + return configs; + } + int workers = Math.max(1, Math.min(concurrency, brokers.size())); + ExecutorService executor = Executors.newFixedThreadPool(workers, runnable -> { + Thread thread = new Thread(runnable, "dbx-rocketmq-consumer-collect"); + thread.setDaemon(true); + return thread; + }); + List>> tasks = new ArrayList<>(brokers.size()); + long brokerTimeout = Math.max(1_000, perBrokerTimeoutMs); + for (String brokerAddr : brokers) { + tasks.add(() -> { + Map partial = new HashMap<>(); + try { + SubscriptionGroupWrapper wrapper = admin.getAllSubscriptionGroup(brokerAddr, brokerTimeout); + if (wrapper != null && wrapper.getSubscriptionGroupTable() != null) { + partial.putAll(wrapper.getSubscriptionGroupTable()); + } + } catch (Exception ignored) { + // Skip unreachable brokers; return whatever others collected. + } + return partial; + }); + } + try { + List>> futures = + executor.invokeAll(tasks, Math.max(1, budgetMs), TimeUnit.MILLISECONDS); + for (Future> future : futures) { + if (future.isCancelled()) { continue; } - for (Map.Entry entry : wrapper.getSubscriptionGroupTable().entrySet()) { - configs.putIfAbsent(entry.getKey(), entry.getValue()); + try { + Map partial = future.get(); + mergeSubscriptionGroupConfigs(configs, partial); + } catch (Exception ignored) { + // Partial page is better than failing the whole list. } - } catch (Exception ignored) { - // Try next broker. } + } finally { + executor.shutdownNow(); } return configs; } @@ -1129,43 +1254,218 @@ public final class RocketMqAgent { /** * DefaultMQAdminExt.examineConsumerConnectionInfo(group) picks a broker from route using * NameServer-registered addresses. Remap to the client-reachable host before querying. + * Live members short-circuit. Empty / 206 offline claims require full remapped coverage and + * complete collision fallbacks (same class of gate as lag/mutate) so a single remapped 206 + * cannot paint memberCount 0 while consumers may be online on an unreached sibling. */ private static ConsumerConnection examineConsumerConnectionInfoRemapped( - DefaultMQAdminExt admin, JsonObject conn, String groupId) { - try { - for (String brokerAddr : resolveMasterBrokerAddrs(admin, conn)) { - try { - ConsumerConnection connection = admin.examineConsumerConnectionInfo(groupId, brokerAddr); - if (connection.getConnectionSet() != null && !connection.getConnectionSet().isEmpty()) { - return connection; - } - } catch (Exception ignored) { - // Try next broker; offline groups return an empty connection below. - } - } - } catch (Exception ignored) { - // No reachable broker addresses. + DefaultMQAdminExt admin, JsonObject conn, String groupId) throws Exception { + MasterBrokerAddrPlan plan = resolveMasterBrokerAddrPlan(admin, conn); + if (plan.isEmpty()) { + throw new MQClientException( + "No reachable RocketMQ master brokers for consumer connection of group " + groupId, null); } - return new ConsumerConnection(); + Exception lastError = null; + int remappedSuccess = 0; + int fallbackSuccess = 0; + ConsumerConnection emptySuccess = null; + for (String brokerAddr : plan.allAddrs()) { + boolean collisionFallback = plan.isCollisionFallback(brokerAddr); + try { + ConsumerConnection connection = admin.examineConsumerConnectionInfo(groupId, brokerAddr); + if (connection.getConnectionSet() != null && !connection.getConnectionSet().isEmpty()) { + return connection; + } + emptySuccess = connection != null ? connection : new ConsumerConnection(); + if (collisionFallback) { + fallbackSuccess++; + } else { + remappedSuccess++; + } + } catch (Exception e) { + // 206 = group has no live clients on that broker — count as probed offline. + if (isConsumerGroupNotOnline(e)) { + if (emptySuccess == null) { + emptySuccess = new ConsumerConnection(); + } + if (collisionFallback) { + fallbackSuccess++; + } else { + remappedSuccess++; + } + continue; + } + lastError = e; + } + } + return requireConsumerConnectionProbeResult( + plan.remappedCount(), + remappedSuccess, + plan.fallbackCount(), + fallbackSuccess, + emptySuccess, + lastError, + groupId + ); + } + + /** + * RocketMQ broker/client code 206: consumer group has no online clients (definitive offline). + */ + static boolean isConsumerGroupNotOnline(Throwable error) { + Throwable cursor = error; + while (cursor != null) { + if (cursor instanceof MQBrokerException brokerEx && brokerEx.getResponseCode() == 206) { + return true; + } + if (cursor instanceof MQClientException clientEx && clientEx.getResponseCode() == 206) { + return true; + } + cursor = cursor.getCause(); + } + return false; + } + + /** + * Accept empty/206 offline only when every remapped master (and every collision fallback) + * answered. Partial coverage must throw so enrich omits memberCount instead of fake 0. + */ + static ConsumerConnection requireConsumerConnectionProbeResult( + int remappedCount, + int remappedSuccess, + int fallbackCount, + int fallbackSuccess, + ConsumerConnection emptySuccess, + Exception lastError, + String groupId + ) throws MQClientException { + if (remappedCount <= 0 && fallbackCount <= 0) { + throw new MQClientException( + "No reachable RocketMQ master brokers for consumer connection of group " + groupId, null); + } + if (remappedSuccess <= 0 && fallbackSuccess <= 0) { + throw new MQClientException( + "Failed to examine consumer connection for group " + groupId + " on all masters", + lastError); + } + if (remappedCount > 0 && remappedSuccess < remappedCount) { + throw new MQClientException( + "Failed to examine consumer connection for group " + groupId + + " on some masters (partial remapped probe)", + lastError); + } + // Docker remap collisions: remapped host:port 206/empty is not enough — sibling masters + // may still have online consumers on unreachable internal addresses. + if (shouldFailClosedOnCollisionPartialMutation(fallbackCount, fallbackSuccess)) { + throw new MQClientException( + "Failed to examine consumer connection for group " + groupId + + " after Docker remap address collision (unreachable sibling masters)", + lastError); + } + return emptySuccess != null ? emptySuccess : new ConsumerConnection(); } private static ConsumeStats examineConsumeStatsRemapped( DefaultMQAdminExt admin, JsonObject conn, String groupId, String topic) throws Exception { + return examineConsumeStatsOnMasters(admin, groupId, topic, resolveMasterBrokerAddrPlan(admin, conn)); + } + + /** + * Probe consume stats on a pre-resolved master plan. Callers that fan out per group + * (lag attach) should resolve the plan once so each task does not re-hit clusterInfo. + */ + private static ConsumeStats examineConsumeStatsOnMasters( + DefaultMQAdminExt admin, String groupId, String topic, MasterBrokerAddrPlan plan) throws Exception { + // RocketMQ 5.3.1: 3-arg examineConsumeStats(clusterName, group, topic) is NOT broker-scoped. + // Must use 4-arg (brokerAddr, group, topic, timeout) or remapped Docker addrs never hit brokers. + long timeout = adminTimeoutMillis(admin); ConsumeStats merged = new ConsumeStats(); - for (String brokerAddr : resolveMasterBrokerAddrs(admin, conn)) { + int remappedSuccess = 0; + int fallbackSuccess = 0; + Exception lastError = null; + for (String brokerAddr : plan.allAddrs()) { + boolean collisionFallback = plan.isCollisionFallback(brokerAddr); try { - ConsumeStats stats = admin.examineConsumeStats(brokerAddr, groupId, topic); + ConsumeStats stats = admin.examineConsumeStats(brokerAddr, groupId, topic, timeout); + if (collisionFallback) { + fallbackSuccess++; + } else { + remappedSuccess++; + } if (stats != null && stats.getOffsetTable() != null) { merged.getOffsetTable().putAll(stats.getOffsetTable()); merged.setConsumeTps(merged.getConsumeTps() + stats.getConsumeTps()); } - } catch (Exception ignored) { - // Try next broker. + } catch (Exception e) { + lastError = e; } } + boolean mergedEmpty = merged.getOffsetTable() == null || merged.getOffsetTable().isEmpty(); + // Remap collisions append Docker-internal originals as best-effort fallbacks. Host-side + // agents usually cannot reach them; those failures must not trip empty-merge fail-closed + // when a remapped master already returned offsets. When remapped probes all fail, gate on + // fallbacks. Empty merge + unreachable collision siblings is still fail-closed: a single + // published host:port can answer for the wrong master and look like healthy zero lag. + if (shouldFailClosedOnCollisionEmpty(mergedEmpty, plan.fallbackCount(), fallbackSuccess)) { + throw new MQClientException( + "Failed to examine consume stats for group " + groupId + + " after Docker remap address collision (unreachable sibling masters)", + lastError); + } + int attemptedForGate; + int successForGate; + if (remappedSuccess > 0) { + attemptedForGate = plan.remappedCount(); + successForGate = remappedSuccess; + } else if (fallbackSuccess > 0) { + attemptedForGate = plan.fallbackCount(); + successForGate = fallbackSuccess; + } else { + attemptedForGate = plan.size(); + successForGate = 0; + } + ensureConsumeStatsProbeSucceeded(attemptedForGate, successForGate, mergedEmpty, lastError, groupId); return merged; } + /** + * Host-side Docker remap collisions collapse N masters onto one published address. An empty + * merge is only trustworthy when every collision-fallback original was also reachable (or + * there were no collisions). + */ + static boolean shouldFailClosedOnCollisionEmpty( + boolean mergedEmpty, int fallbackCount, int fallbackSuccess) { + return mergedEmpty && fallbackCount > 0 && fallbackSuccess < fallbackCount; + } + + /** + * Fail closed when no master answered, or when some masters failed and the merge is empty + * (partial outage otherwise looks like healthy zero lag). + */ + static void ensureConsumeStatsProbeSucceeded( + int attemptedBrokers, + int successCount, + boolean mergedEmpty, + Exception lastError, + String groupId + ) throws MQClientException { + if (attemptedBrokers <= 0) { + throw new MQClientException( + "No reachable RocketMQ master brokers for consume stats of group " + groupId, null); + } + if (successCount <= 0) { + throw new MQClientException( + "Failed to examine consume stats for group " + groupId + " on all masters", + lastError); + } + if (mergedEmpty && successCount < attemptedBrokers) { + throw new MQClientException( + "Failed to examine consume stats for group " + groupId + + " on some masters (empty merge after partial failure)", + lastError); + } + } + /** * RocketMQ admin {@code queryTopicConsumeByWho(topic)} contacts the first broker using * NameServer-registered addresses (often Docker-internal IPs). Remap each broker before @@ -1213,16 +1513,8 @@ public final class RocketMqAgent { private static GroupList queryTopicConsumeByWhoOnBroker( DefaultMQAdminExt admin, String brokerAddr, String topic) throws Exception { try { - var implField = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl"); - implField.setAccessible(true); - Object impl = implField.get(admin); - var mqClientMethod = impl.getClass().getMethod("getMqClientInstance"); - Object mqClient = mqClientMethod.invoke(impl); - var apiMethod = mqClient.getClass().getMethod("getMQClientAPIImpl"); - Object api = apiMethod.invoke(mqClient); - var timeoutField = impl.getClass().getDeclaredField("timeoutMillis"); - timeoutField.setAccessible(true); - long timeout = timeoutField.getLong(impl); + Object api = mqClientApiImpl(admin); + long timeout = adminTimeoutMillis(admin); var queryMethod = api.getClass().getMethod( "queryTopicConsumeByWho", String.class, String.class, long.class); return (GroupList) queryMethod.invoke(api, brokerAddr, topic, timeout); @@ -1289,12 +1581,17 @@ public final class RocketMqAgent { if (limit <= 0) { limit = DEFAULT_LIST_LIMIT; } + boolean includeLag = boolOrDefault(params, "includeLag", false); - Map configs = collectConsumerGroupConfigs(admin, conn); + Map configs; Set groups = new TreeSet<>(); if (!topicFilter.isBlank()) { + // Dashboard-style: discover groups via topic; skip full-cluster subscription dump. groups.addAll(queryTopicConsumeByWhoRemapped(admin, conn, topicFilter)); + // Merge every reachable master dump so FIFO typing is not stuck on the first broker. + configs = collectConsumerGroupConfigsForTopicFilter(admin, conn); } else { + configs = collectConsumerGroupConfigs(admin, conn); groups.addAll(configs.keySet()); } @@ -1307,7 +1604,8 @@ public final class RocketMqAgent { row.put("groupId", groupId); row.put("state", "UNKNOWN"); row.put("simpleGroup", false); - row.put("groupType", "NORMAL"); + // Placeholder until config classify runs; missing dump → UNKNOWN, not NORMAL. + row.put("groupType", "UNKNOWN"); row.put("messageModel", "CLUSTERING"); rows.add(row); } @@ -1322,6 +1620,9 @@ public final class RocketMqAgent { if (boolOrDefault(params, "enrich", false)) { enrichConsumerGroupRows(admin, conn, page); } + if (includeLag && !topicFilter.isBlank()) { + attachConsumerLags(admin, conn, page, topicFilter); + } Map result = new LinkedHashMap<>(); result.put("groups", page); @@ -1331,6 +1632,80 @@ public final class RocketMqAgent { return result; } + /** Concurrent lag lookup for the current page; budget-capped like enrich. */ + private static void attachConsumerLags( + DefaultMQAdminExt admin, + JsonObject conn, + List> page, + String topic + ) { + if (page.isEmpty() || topic == null || topic.isBlank()) { + return; + } + final MasterBrokerAddrPlan masterPlan; + try { + // Resolve/remap masters once — per-group examineBrokerClusterInfo burns the lag budget. + masterPlan = resolveMasterBrokerAddrPlan(admin, conn); + } catch (Exception e) { + for (Map row : page) { + row.put("totalLagFailed", true); + } + return; + } + int workers = Math.max(1, Math.min(CONSUMER_LAG_CONCURRENCY, page.size())); + ExecutorService executor = Executors.newFixedThreadPool(workers, runnable -> { + Thread thread = new Thread(runnable, "dbx-rocketmq-consumer-lag"); + thread.setDaemon(true); + return thread; + }); + // Nullable Long: omit totalLag on failure; mark totalLagFailed so list UI does not show 0. + List> tasks = new ArrayList<>(page.size()); + for (Map row : page) { + String groupId = String.valueOf(row.get("groupId")); + tasks.add(() -> { + // examineConsumeStatsOnMasters throws when no master succeeds. + ConsumeStats stats = examineConsumeStatsOnMasters(admin, groupId, topic, masterPlan); + long totalLag = 0; + if (stats.getOffsetTable() != null) { + for (var entry : stats.getOffsetTable().entrySet()) { + totalLag += Math.max(0, entry.getValue().getBrokerOffset() - entry.getValue().getConsumerOffset()); + } + } + return totalLag; + }); + } + try { + List> results = + executor.invokeAll(tasks, Math.max(1, CONSUMER_LAG_BUDGET_MS), TimeUnit.MILLISECONDS); + for (int i = 0; i < page.size(); i++) { + Future future = i < results.size() ? results.get(i) : null; + if (future == null || future.isCancelled()) { + page.get(i).put("totalLagFailed", true); + continue; + } + try { + Long lag = future.get(); + if (lag != null) { + page.get(i).put("totalLag", lag); + } else { + page.get(i).put("totalLagFailed", true); + } + } catch (Exception ignored) { + page.get(i).put("totalLagFailed", true); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + for (Map row : page) { + if (!row.containsKey("totalLag")) { + row.put("totalLagFailed", true); + } + } + } finally { + executor.shutdownNow(); + } + } + @FunctionalInterface interface ConsumerConnectionLookup { ConsumerConnection load(String groupId) throws Exception; @@ -1381,7 +1756,7 @@ public final class RocketMqAgent { // Keep the base group row when enrichment fails. } } - row.putIfAbsent("memberCount", 0); + // Omit memberCount on cancel/timeout/failure so UI shows '-' instead of fake offline 0. row.putIfAbsent("topics", Collections.emptyList()); } } @@ -1407,7 +1782,7 @@ public final class RocketMqAgent { } enrichment.put("topics", topics); } catch (Exception ignored) { - enrichment.put("memberCount", 0); + // Leave memberCount unset — probe did not finish; list UI must not show 0. enrichment.put("topics", Collections.emptyList()); } return enrichment; @@ -1454,8 +1829,16 @@ public final class RocketMqAgent { private static Object deleteConsumerGroup(JsonObject params) throws Exception { DefaultMQAdminExt admin = requireAdmin(); + JsonObject conn = connectionObject(params); String groupId = requireString(params, "groupId"); - admin.deleteSubscriptionGroup(brokerAddr(params, admin), groupId); + // Dashboard deletes per broker; wipe every reachable master so the group does not reappear. + mutateSubscriptionGroupOnMasters( + admin, + conn, + groupId, + "delete", + brokerAddr -> admin.deleteSubscriptionGroup(brokerAddr, groupId) + ); return Collections.singletonMap("ok", true); } @@ -1480,14 +1863,103 @@ public final class RocketMqAgent { config.setGroupName(groupId); } applySubscriptionGroupConfigUpdates(config, params); - for (String brokerAddr : resolveMasterBrokerAddrs(admin, conn)) { + SubscriptionGroupConfig toWrite = config; + mutateSubscriptionGroupOnMasters( + admin, + conn, + groupId, + "update", + brokerAddr -> admin.createAndUpdateSubscriptionGroupConfig(brokerAddr, toWrite) + ); + return Collections.singletonMap("ok", true); + } + + @FunctionalInterface + interface BrokerSubscriptionMutation { + void apply(String brokerAddr) throws Exception; + } + + /** + * Apply a subscription-group mutation on every remapped master (and collision fallbacks). + * Require full coverage of remapped addrs; Docker remap collisions must also succeed on + * fallback originals — a single published host:port hit must not report ok while sibling + * masters still hold the old group/config. + */ + static void mutateSubscriptionGroupOnMasters( + DefaultMQAdminExt admin, + JsonObject conn, + String groupId, + String action, + BrokerSubscriptionMutation mutation + ) throws Exception { + MasterBrokerAddrPlan plan = resolveMasterBrokerAddrPlan(admin, conn); + int remappedSuccess = 0; + int fallbackSuccess = 0; + Exception lastError = null; + for (String brokerAddr : plan.allAddrs()) { try { - admin.createAndUpdateSubscriptionGroupConfig(brokerAddr, config); - } catch (Exception ignored) { - // Try next broker when Docker/internal broker addresses are unreachable. + mutation.apply(brokerAddr); + if (plan.isCollisionFallback(brokerAddr)) { + fallbackSuccess++; + } else { + remappedSuccess++; + } + } catch (Exception e) { + lastError = e; } } - return Collections.singletonMap("ok", true); + ensureSubscriptionGroupMutationSucceeded( + plan.remappedCount(), + remappedSuccess, + plan.fallbackCount(), + fallbackSuccess, + lastError, + action, + groupId + ); + } + + /** + * Gate subscription-group mutate/delete. Partial remapped updates and Docker collision + * fallbacks that never answered must fail closed (group would otherwise reappear / diverge). + */ + static void ensureSubscriptionGroupMutationSucceeded( + int remappedCount, + int remappedSuccess, + int fallbackCount, + int fallbackSuccess, + Exception lastError, + String action, + String groupId + ) throws MQClientException { + if (remappedCount <= 0 && fallbackCount <= 0) { + throw new MQClientException( + "No reachable RocketMQ master brokers for " + action + " of group " + groupId, null); + } + if (remappedSuccess <= 0 && fallbackSuccess <= 0) { + throw new MQClientException( + "Failed to " + action + " consumer group " + groupId + " on all masters", + lastError); + } + if (remappedCount > 0 && remappedSuccess < remappedCount) { + throw new MQClientException( + "Failed to " + action + " consumer group " + groupId + + " on some masters (partial remapped update)", + lastError); + } + // Remap collisions collapse N masters onto one published addr. Updating that addr once + // leaves sibling masters untouched unless collision-fallback originals also succeed. + if (shouldFailClosedOnCollisionPartialMutation(fallbackCount, fallbackSuccess)) { + throw new MQClientException( + "Failed to " + action + " consumer group " + groupId + + " after Docker remap address collision (unreachable sibling masters)", + lastError); + } + } + + /** True when collision fallbacks exist but were not all successfully mutated. */ + static boolean shouldFailClosedOnCollisionPartialMutation(int fallbackCount, int fallbackSuccess) { + return fallbackCount > 0 && fallbackSuccess < fallbackCount; } private static Map subscriptionGroupConfigToMap(SubscriptionGroupConfig config) { @@ -1566,25 +2038,46 @@ public final class RocketMqAgent { String groupId = requireString(params, "groupId"); String topic = requireString(params, "topic"); ConsumeStats stats = examineConsumeStatsRemapped(admin, conn, groupId, topic); + // Best-effort queue→client mapping (Dashboard getClientConnection); failures must not hide offsets. + Map queueClientMap = resolveQueueClientMap(admin, conn, groupId); + return buildConsumerLagResult(stats, queueClientMap); + } + /** + * Build lag payload from ConsumeStats. Keeps Kafka-compatible field names + * ({@code currentOffset}/{@code endOffset}/{@code lag}) and adds Dashboard fields + * ({@code brokerName}/{@code lastTimestamp}/{@code consumerClient}). + */ + static Map buildConsumerLagResult( + ConsumeStats stats, Map queueClientMap) { long totalLag = 0; List> partitions = new ArrayList<>(); - if (stats.getOffsetTable() != null) { + if (stats != null && stats.getOffsetTable() != null) { for (var entry : stats.getOffsetTable().entrySet()) { - long consumerOffset = entry.getValue().getConsumerOffset(); - long brokerOffset = entry.getValue().getBrokerOffset(); + MessageQueue mq = entry.getKey(); + OffsetWrapper offset = entry.getValue(); + long consumerOffset = offset.getConsumerOffset(); + long brokerOffset = offset.getBrokerOffset(); long lag = Math.max(0, brokerOffset - consumerOffset); totalLag += lag; Map partition = new LinkedHashMap<>(); - partition.put("partition", entry.getKey().getQueueId()); + partition.put("partition", mq.getQueueId()); partition.put("currentOffset", consumerOffset); partition.put("endOffset", brokerOffset); partition.put("lag", lag); + // Dashboard-compatible consume-detail columns. + partition.put("brokerName", mq.getBrokerName() != null ? mq.getBrokerName() : ""); + partition.put("lastTimestamp", offset.getLastTimestamp()); + String clientId = queueClientMap != null ? queueClientMap.get(mq) : null; + partition.put("consumerClient", clientId != null ? clientId : ""); partitions.add(partition); } } - partitions.sort(Comparator.comparingInt(a -> (int) a.get("partition"))); + // Sort by broker then queue so multi-broker topics stay readable. + partitions.sort(Comparator + .comparing((Map a) -> String.valueOf(a.get("brokerName"))) + .thenComparingInt(a -> (int) a.get("partition"))); Map result = new LinkedHashMap<>(); result.put("partitions", partitions); @@ -1592,6 +2085,142 @@ public final class RocketMqAgent { return result; } + /** + * Map MessageQueue → online consumer clientId via ConsumerRunningInfo.mqTable. + * Mirrors RocketMQ Dashboard ConsumerServiceImpl#getClientConnection; swallows errors. + */ + static Map resolveQueueClientMap( + DefaultMQAdminExt admin, JsonObject conn, String groupId) { + Map results = new HashMap<>(); + try { + ConsumerConnection connection = examineConsumerConnectionInfoRemapped(admin, conn, groupId); + if (connection.getConnectionSet() == null || connection.getConnectionSet().isEmpty()) { + return results; + } + for (Connection clientConn : connection.getConnectionSet()) { + String clientId = clientConn.getClientId(); + if (clientId == null || clientId.isBlank()) { + continue; + } + try { + // DefaultMQAdminExt picks the first NameServer-registered broker (often Docker-internal). + ConsumerRunningInfo runningInfo = + getConsumerRunningInfoRemapped(admin, conn, groupId, clientId, false); + if (runningInfo == null || runningInfo.getMqTable() == null) { + continue; + } + for (MessageQueue mq : runningInfo.getMqTable().keySet()) { + results.put(mq, clientId); + } + } catch (Exception ignored) { + // Offline/unreachable client: keep offsets without client assignment. + } + } + } catch (Exception ignored) { + // Connection lookup failed; return empty map. + } + return results; + } + + /** + * Like {@link DefaultMQAdminExt#getConsumerRunningInfo}, but remaps broker addresses for + * Docker/host-published ports before invoking {@code MQClientAPIImpl}. + */ + static ConsumerRunningInfo getConsumerRunningInfoRemapped( + DefaultMQAdminExt admin, + JsonObject conn, + String groupId, + String clientId, + boolean jstack + ) { + // Upstream walks %RETRY%{group} route and contacts the first broker addr as registered. + try { + String retryTopic = MixAll.RETRY_GROUP_TOPIC_PREFIX + groupId; + TopicRouteData route = admin.examineTopicRouteInfo(retryTopic); + if (route != null && route.getBrokerDatas() != null) { + for (BrokerData brokerData : route.getBrokerDatas()) { + String rawAddr = brokerData.selectBrokerAddr(); + if (rawAddr == null || rawAddr.isBlank()) { + continue; + } + String brokerAddr = remapBrokerAddrForClient(rawAddr, conn); + try { + ConsumerRunningInfo info = + getConsumerRunningInfoOnBroker(admin, brokerAddr, groupId, clientId, jstack); + if (info != null) { + return info; + } + } catch (Exception ignored) { + // Try next remapped broker. + } + } + } + } catch (Exception ignored) { + // Fall through to master-broker scan. + } + try { + for (String brokerAddr : resolveMasterBrokerAddrs(admin, conn)) { + try { + ConsumerRunningInfo info = + getConsumerRunningInfoOnBroker(admin, brokerAddr, groupId, clientId, jstack); + if (info != null) { + return info; + } + } catch (Exception ignored) { + // Try next master. + } + } + } catch (Exception ignored) { + // No reachable masters. + } + return null; + } + + /** Broker-scoped ConsumerRunningInfo; DefaultMQAdminExt has no remapped overload. */ + private static ConsumerRunningInfo getConsumerRunningInfoOnBroker( + DefaultMQAdminExt admin, + String brokerAddr, + String groupId, + String clientId, + boolean jstack + ) throws Exception { + try { + Object api = mqClientApiImpl(admin); + long timeout = adminTimeoutMillis(admin); + var method = api.getClass().getMethod( + "getConsumerRunningInfo", + String.class, + String.class, + String.class, + boolean.class, + long.class + ); + return (ConsumerRunningInfo) method.invoke(api, brokerAddr, groupId, clientId, jstack, timeout); + } catch (ReflectiveOperationException e) { + throw new MQClientException( + "Failed to get consumer running info on broker " + brokerAddr, e); + } + } + + private static Object mqClientApiImpl(DefaultMQAdminExt admin) throws ReflectiveOperationException { + var implField = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl"); + implField.setAccessible(true); + Object impl = implField.get(admin); + var mqClientMethod = impl.getClass().getMethod("getMqClientInstance"); + Object mqClient = mqClientMethod.invoke(impl); + var apiMethod = mqClient.getClass().getMethod("getMQClientAPIImpl"); + return apiMethod.invoke(mqClient); + } + + private static long adminTimeoutMillis(DefaultMQAdminExt admin) throws ReflectiveOperationException { + var implField = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl"); + implField.setAccessible(true); + Object impl = implField.get(admin); + var timeoutField = impl.getClass().getDeclaredField("timeoutMillis"); + timeoutField.setAccessible(true); + return timeoutField.getLong(impl); + } + private static Object listProducers(JsonObject params) throws Exception { DefaultMQAdminExt admin = requireAdmin(); JsonObject conn = connectionObject(params); @@ -1910,9 +2539,10 @@ public final class RocketMqAgent { static DefaultMQAdminExt buildAdminClient(JsonObject conn) throws Exception { long timeoutMs = intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS); RPCHook rpcHook = buildRpcHook(conn); + // Always pass timeout — DefaultMQAdminExt() ignores request_timeout_ms. DefaultMQAdminExt admin = rpcHook != null ? new DefaultMQAdminExt(rpcHook, timeoutMs) - : new DefaultMQAdminExt(); + : new DefaultMQAdminExt(timeoutMs); admin.setNamesrvAddr(namesrvAddr(conn)); admin.setAdminExtGroup("_DBX_ROCKETMQ_ADMIN_" + UUID.randomUUID()); admin.setInstanceName("DBX_" + UUID.randomUUID()); @@ -2016,6 +2646,11 @@ public final class RocketMqAgent { return resolveMasterBrokerAddrs(admin, conn, null); } + private static MasterBrokerAddrPlan resolveMasterBrokerAddrPlan( + DefaultMQAdminExt admin, JsonObject conn) throws Exception { + return resolveMasterBrokerAddrPlan(admin, conn, null, admin.examineBrokerClusterInfo()); + } + static List resolveMasterBrokerAddrs( DefaultMQAdminExt admin, JsonObject conn, String brokerNameFilter) throws Exception { return resolveMasterBrokerAddrs(admin, conn, brokerNameFilter, admin.examineBrokerClusterInfo()); @@ -2024,20 +2659,27 @@ public final class RocketMqAgent { static List resolveMasterBrokerAddrs( DefaultMQAdminExt admin, JsonObject conn, String brokerNameFilter, ClusterInfo clusterInfo) throws Exception { - LinkedHashSet addrs = masterBrokerAddrsFromClusterInfo(clusterInfo, conn, brokerNameFilter); - if (addrs.isEmpty()) { - addrs.add(resolveBrokerAddr(admin, conn, clusterInfo)); + return new ArrayList<>(resolveMasterBrokerAddrPlan(admin, conn, brokerNameFilter, clusterInfo).allAddrs()); + } + + static MasterBrokerAddrPlan resolveMasterBrokerAddrPlan( + DefaultMQAdminExt admin, JsonObject conn, String brokerNameFilter, ClusterInfo clusterInfo) + throws Exception { + MasterBrokerAddrPlan plan = masterBrokerAddrsFromClusterInfo(clusterInfo, conn, brokerNameFilter); + if (plan.isEmpty()) { + plan.addRemapped(resolveBrokerAddr(admin, conn, clusterInfo)); } - return new ArrayList<>(addrs); + return plan; } /** Extract remapped master broker addresses from a ClusterInfo snapshot. */ - static LinkedHashSet masterBrokerAddrsFromClusterInfo( + static MasterBrokerAddrPlan masterBrokerAddrsFromClusterInfo( ClusterInfo clusterInfo, JsonObject conn, String brokerNameFilter) { - LinkedHashSet addrs = new LinkedHashSet<>(); + MasterBrokerAddrPlan plan = new MasterBrokerAddrPlan(); if (clusterInfo == null || clusterInfo.getBrokerAddrTable() == null) { - return addrs; + return plan; } + LinkedHashSet remappedSeen = new LinkedHashSet<>(); for (BrokerData broker : clusterInfo.getBrokerAddrTable().values()) { if (brokerNameFilter != null && !brokerNameFilter.isBlank() && !brokerNameFilter.equals(broker.getBrokerName())) { @@ -2050,11 +2692,67 @@ public final class RocketMqAgent { if (masterAddr == null || masterAddr.isBlank()) { masterAddr = broker.selectBrokerAddr(); } - if (masterAddr != null && !masterAddr.isBlank()) { - addrs.add(remapBrokerAddrForClient(masterAddr, conn)); + if (masterAddr == null || masterAddr.isBlank()) { + continue; + } + String remapped = remapBrokerAddrForClient(masterAddr, conn); + // Multi-broker Docker often remaps every master to the same host:port. Keep the + // original address for collisions so in-network agents can still reach each broker. + if (remappedSeen.add(remapped)) { + plan.addRemapped(remapped); + } else if (!remapped.equals(masterAddr)) { + plan.addCollisionFallback(masterAddr); } } - return addrs; + return plan; + } + + /** + * Remapped masters plus optional collision-fallback originals. Fail-closed consume-stats + * gating treats fallbacks as best-effort so host-side Docker agents do not mark lag + * unavailable when only unreachable internal IPs fail. + */ + static final class MasterBrokerAddrPlan { + private final LinkedHashSet remapped = new LinkedHashSet<>(); + private final LinkedHashSet collisionFallbacks = new LinkedHashSet<>(); + + void addRemapped(String addr) { + if (addr != null && !addr.isBlank()) { + remapped.add(addr); + } + } + + void addCollisionFallback(String addr) { + if (addr != null && !addr.isBlank() && !remapped.contains(addr)) { + collisionFallbacks.add(addr); + } + } + + boolean isCollisionFallback(String addr) { + return collisionFallbacks.contains(addr); + } + + LinkedHashSet allAddrs() { + LinkedHashSet all = new LinkedHashSet<>(remapped); + all.addAll(collisionFallbacks); + return all; + } + + int remappedCount() { + return remapped.size(); + } + + int fallbackCount() { + return collisionFallbacks.size(); + } + + int size() { + return remapped.size() + collisionFallbacks.size(); + } + + boolean isEmpty() { + return remapped.isEmpty() && collisionFallbacks.isEmpty(); + } } private static void applyTopicConfigValue(TopicConfig config, String key, String value) { @@ -2490,20 +3188,49 @@ public final class RocketMqAgent { return true; } - private static boolean isLikelyUnreachableBrokerHost(String host) { + /** Package-visible for tests; RFC1918 + common Docker DNS names. */ + static boolean isLikelyUnreachableBrokerHost(String host) { if (host == null || host.isBlank()) { return false; } if ("127.0.0.1".equals(host) || "localhost".equalsIgnoreCase(host) || "::1".equals(host)) { return false; } - return host.startsWith("172.") - || host.startsWith("10.") - || host.startsWith("192.168.") + // RFC1918: 10/8, 172.16/12, 192.168/16 — not every 172.* address. + return isRfc1918PrivateIpv4(host) || host.endsWith(".docker") || host.contains(".docker."); } + static boolean isRfc1918PrivateIpv4(String host) { + if (host == null || host.isBlank()) { + return false; + } + String[] parts = host.split("\\."); + if (parts.length != 4) { + return false; + } + int[] octets = new int[4]; + for (int i = 0; i < 4; i++) { + try { + octets[i] = Integer.parseInt(parts[i]); + } catch (NumberFormatException e) { + return false; + } + if (octets[i] < 0 || octets[i] > 255) { + return false; + } + } + if (octets[0] == 10) { + return true; + } + if (octets[0] == 192 && octets[1] == 168) { + return true; + } + // 172.16.0.0 – 172.31.255.255 + return octets[0] == 172 && octets[1] >= 16 && octets[1] <= 31; + } + private static List> brokerNodes(ClusterInfo clusterInfo) { List> brokers = new ArrayList<>(); int id = 0; diff --git a/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java b/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java index 6140b55d3..a10d9e17f 100644 --- a/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java +++ b/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java @@ -2,14 +2,18 @@ package com.dbx.agent.rocketmq; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; import org.apache.rocketmq.remoting.protocol.body.ProducerInfo; @@ -23,6 +27,7 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -129,6 +134,21 @@ class RocketMqAgentTest { assertEquals("127.0.0.1:10911", RocketMqAgent.remapBrokerAddrForClient("172.18.0.3:10911", conn)); assertEquals("127.0.0.1:10911", RocketMqAgent.remapBrokerAddrForClient("10.0.0.5:10911", conn)); assertEquals("broker.example.com:10911", RocketMqAgent.remapBrokerAddrForClient("broker.example.com:10911", conn)); + // 172.15/172.32 are not RFC1918 — leave reachable public/non-private hosts alone. + assertEquals("172.15.0.1:10911", RocketMqAgent.remapBrokerAddrForClient("172.15.0.1:10911", conn)); + assertEquals("172.32.0.1:10911", RocketMqAgent.remapBrokerAddrForClient("172.32.0.1:10911", conn)); + } + + @Test + void isRfc1918PrivateIpv4MatchesOnlyPrivateRanges() { + assertTrue(RocketMqAgent.isRfc1918PrivateIpv4("10.0.0.1")); + assertTrue(RocketMqAgent.isRfc1918PrivateIpv4("172.16.0.1")); + assertTrue(RocketMqAgent.isRfc1918PrivateIpv4("172.31.255.255")); + assertTrue(RocketMqAgent.isRfc1918PrivateIpv4("192.168.1.1")); + assertFalse(RocketMqAgent.isRfc1918PrivateIpv4("172.15.0.1")); + assertFalse(RocketMqAgent.isRfc1918PrivateIpv4("172.32.0.1")); + assertFalse(RocketMqAgent.isRfc1918PrivateIpv4("8.8.8.8")); + assertFalse(RocketMqAgent.isRfc1918PrivateIpv4("broker.example.com")); } @Test @@ -255,6 +275,8 @@ class RocketMqAgentTest { SubscriptionGroupConfig normal = new SubscriptionGroupConfig(); normal.setConsumeMessageOrderly(false); assertEquals("NORMAL", RocketMqAgent.classifyConsumerGroupType("MyGroup", normal)); + // Missing dump must not look like NORMAL (FIFO groups would be mislabeled). + assertEquals("UNKNOWN", RocketMqAgent.classifyConsumerGroupType("MissingConfigGroup", null)); } @Test @@ -291,13 +313,94 @@ class RocketMqAgentTest { assertTrue(maxActive.get() > 1); for (Map row : rows) { + // Successful empty connection probe → genuine offline 0. assertEquals(0, row.get("memberCount")); assertEquals(List.of(), row.get("topics")); } } @Test - void enrichConsumerGroupRowsReturnsDefaultsWhenBudgetExpires() { + void requireConsumerConnectionProbeResultFailsClosedWhenNoMasterAnswers() throws Exception { + assertThrows( + MQClientException.class, + () -> RocketMqAgent.requireConsumerConnectionProbeResult( + 1, 0, 0, 0, null, new IllegalStateException("timeout"), "g1") + ); + ConsumerConnection offline = new ConsumerConnection(); + assertSame( + offline, + RocketMqAgent.requireConsumerConnectionProbeResult(1, 1, 0, 0, offline, null, "g1") + ); + } + + @Test + void requireConsumerConnectionProbeResultFailsClosedOnRemapped206WithUnreachableFallback() { + Exception cause = new IllegalStateException("fallback unreachable"); + MQClientException err = assertThrows( + MQClientException.class, + () -> RocketMqAgent.requireConsumerConnectionProbeResult( + 1, 1, 1, 0, new ConsumerConnection(), cause, "g1") + ); + assertTrue(err.getMessage().contains("Docker remap address collision")); + assertSame(cause, err.getCause()); + } + + @Test + void requireConsumerConnectionProbeResultFailsClosedOnPartialRemappedOffline() { + Exception cause = new IllegalStateException("broker-b timeout"); + MQClientException err = assertThrows( + MQClientException.class, + () -> RocketMqAgent.requireConsumerConnectionProbeResult( + 2, 1, 0, 0, new ConsumerConnection(), cause, "g1") + ); + assertTrue(err.getMessage().contains("partial remapped probe")); + assertSame(cause, err.getCause()); + } + + @Test + void requireConsumerConnectionProbeResultAllowsFullOfflineCoverage() throws Exception { + ConsumerConnection offline = new ConsumerConnection(); + assertSame( + offline, + RocketMqAgent.requireConsumerConnectionProbeResult(2, 2, 0, 0, offline, null, "g1") + ); + assertSame( + offline, + RocketMqAgent.requireConsumerConnectionProbeResult(1, 1, 1, 1, offline, null, "g1") + ); + } + + @Test + void isConsumerGroupNotOnlineDetectsBrokerCode206() { + assertTrue(RocketMqAgent.isConsumerGroupNotOnline( + new MQBrokerException(206, "the consumer group[g1] not online"))); + assertTrue(RocketMqAgent.isConsumerGroupNotOnline( + new MQClientException(206, "the consumer group[g1] not online"))); + assertTrue(RocketMqAgent.isConsumerGroupNotOnline( + new RuntimeException(new MQBrokerException(206, "not online")))); + assertFalse(RocketMqAgent.isConsumerGroupNotOnline( + new MQBrokerException(1, "system error"))); + assertFalse(RocketMqAgent.isConsumerGroupNotOnline(new IllegalStateException("timeout"))); + } + + @Test + void enrichConsumerGroupRowsOmitsMemberCountWhenConnectionProbeFails() { + List> rows = new ArrayList<>(); + Map row = new LinkedHashMap<>(); + row.put("groupId", "unreachable-group"); + rows.add(row); + + RocketMqAgent.enrichConsumerGroupRows(rows, groupId -> { + throw new MQClientException("Failed to examine consumer connection on all masters", null); + }, 1, 1_000); + + // All-master connection probe failure must not look like offline 0. + assertFalse(row.containsKey("memberCount")); + assertEquals(List.of(), row.get("topics")); + } + + @Test + void enrichConsumerGroupRowsOmitsMemberCountWhenBudgetExpires() { List> rows = IntStream.range(0, 4) .mapToObj(index -> { Map row = new LinkedHashMap<>(); @@ -316,7 +419,8 @@ class RocketMqAgentTest { long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); assertTrue(elapsedMs < 1_000, "enrichment exceeded its response budget: " + elapsedMs + "ms"); for (Map row : rows) { - assertEquals(0, row.get("memberCount")); + // Cancelled probes must not look offline (0); UI shows '-' when memberCount is absent. + assertFalse(row.containsKey("memberCount")); assertEquals(List.of(), row.get("topics")); } } @@ -575,4 +679,230 @@ class RocketMqAgentTest { assertTrue(RocketMqAgent.connectionMatches(base, same)); assertFalse(RocketMqAgent.connectionMatches(base, differentNamesrv)); } + + @Test + void buildConsumerLagResultIncludesBrokerClientAndTimestamp() { + MessageQueue mq0 = new MessageQueue("TX_TOPIC", "broker-a", 0); + MessageQueue mq1 = new MessageQueue("TX_TOPIC", "broker-a", 1); + OffsetWrapper offset0 = new OffsetWrapper(); + offset0.setBrokerOffset(100); + offset0.setConsumerOffset(90); + offset0.setLastTimestamp(1_725_000_000_000L); + OffsetWrapper offset1 = new OffsetWrapper(); + offset1.setBrokerOffset(50); + offset1.setConsumerOffset(50); + offset1.setLastTimestamp(0L); + + ConsumeStats stats = new ConsumeStats(); + stats.getOffsetTable().put(mq0, offset0); + stats.getOffsetTable().put(mq1, offset1); + + Map clients = new HashMap<>(); + clients.put(mq0, "172.18.2.212@7#1"); + + Map result = RocketMqAgent.buildConsumerLagResult(stats, clients); + assertEquals(10L, result.get("totalLag")); + + @SuppressWarnings("unchecked") + List> partitions = (List>) result.get("partitions"); + assertEquals(2, partitions.size()); + assertEquals("broker-a", partitions.get(0).get("brokerName")); + assertEquals(0, partitions.get(0).get("partition")); + assertEquals(90L, partitions.get(0).get("currentOffset")); + assertEquals(100L, partitions.get(0).get("endOffset")); + assertEquals(10L, partitions.get(0).get("lag")); + assertEquals(1_725_000_000_000L, partitions.get(0).get("lastTimestamp")); + assertEquals("172.18.2.212@7#1", partitions.get(0).get("consumerClient")); + // Missing client mapping still returns offsets with empty consumerClient. + assertEquals("", partitions.get(1).get("consumerClient")); + assertEquals(0L, partitions.get(1).get("lastTimestamp")); + } + + @Test + void buildConsumerLagResultHandlesNullStatsAndEmptyClientMap() { + Map empty = RocketMqAgent.buildConsumerLagResult(null, null); + assertEquals(0L, empty.get("totalLag")); + @SuppressWarnings("unchecked") + List> partitions = (List>) empty.get("partitions"); + assertTrue(partitions.isEmpty()); + } + + @Test + void mergeSubscriptionGroupConfigsPrefersFifoAndKeepsLaterOnlyGroups() { + Map merged = new LinkedHashMap<>(); + + SubscriptionGroupConfig normalA = new SubscriptionGroupConfig(); + normalA.setGroupName("group-a"); + normalA.setConsumeMessageOrderly(false); + + SubscriptionGroupConfig fifoA = new SubscriptionGroupConfig(); + fifoA.setGroupName("group-a"); + fifoA.setConsumeMessageOrderly(true); + + SubscriptionGroupConfig onlyOnLater = new SubscriptionGroupConfig(); + onlyOnLater.setGroupName("group-b"); + onlyOnLater.setConsumeMessageOrderly(true); + + // Intermediate broker repeats group-a as NORMAL (size would not grow), then later adds FIFO-only group-b. + RocketMqAgent.mergeSubscriptionGroupConfigs(merged, Map.of("group-a", normalA)); + RocketMqAgent.mergeSubscriptionGroupConfigs(merged, Map.of("group-a", normalA)); + RocketMqAgent.mergeSubscriptionGroupConfigs( + merged, + Map.of("group-a", fifoA, "group-b", onlyOnLater) + ); + + assertEquals(2, merged.size()); + assertTrue(merged.get("group-a").isConsumeMessageOrderly()); + assertTrue(merged.get("group-b").isConsumeMessageOrderly()); + assertEquals("FIFO", RocketMqAgent.classifyConsumerGroupType("group-a", merged.get("group-a"))); + assertEquals("FIFO", RocketMqAgent.classifyConsumerGroupType("group-b", merged.get("group-b"))); + } + + @Test + void mergeSubscriptionGroupConfigsDoesNotDowngradeFifoToNormal() { + Map merged = new LinkedHashMap<>(); + SubscriptionGroupConfig fifo = new SubscriptionGroupConfig(); + fifo.setGroupName("ordered"); + fifo.setConsumeMessageOrderly(true); + SubscriptionGroupConfig normal = new SubscriptionGroupConfig(); + normal.setGroupName("ordered"); + normal.setConsumeMessageOrderly(false); + + RocketMqAgent.mergeSubscriptionGroupConfigs(merged, Map.of("ordered", fifo)); + RocketMqAgent.mergeSubscriptionGroupConfigs(merged, Map.of("ordered", normal)); + + assertTrue(merged.get("ordered").isConsumeMessageOrderly()); + } + + @Test + void ensureConsumeStatsProbeSucceededRejectsAllBrokerFailures() { + MQClientException noMasters = assertThrows( + MQClientException.class, + () -> RocketMqAgent.ensureConsumeStatsProbeSucceeded(0, 0, true, null, "GID_A")); + assertTrue(noMasters.getMessage().contains("No reachable RocketMQ master")); + + Exception cause = new RuntimeException("broker down"); + MQClientException allFailed = assertThrows( + MQClientException.class, + () -> RocketMqAgent.ensureConsumeStatsProbeSucceeded(2, 0, true, cause, "GID_A")); + assertTrue(allFailed.getMessage().contains("Failed to examine consume stats")); + assertSame(cause, allFailed.getCause()); + } + + @Test + void ensureConsumeStatsProbeSucceededRejectsPartialFailureWithEmptyMerge() { + Exception cause = new RuntimeException("one broker down"); + MQClientException partialEmpty = assertThrows( + MQClientException.class, + () -> RocketMqAgent.ensureConsumeStatsProbeSucceeded(3, 1, true, cause, "GID_A")); + assertTrue(partialEmpty.getMessage().contains("partial failure")); + assertSame(cause, partialEmpty.getCause()); + } + + @Test + void ensureConsumeStatsProbeSucceededAllowsEmptyOffsetsWhenAllBrokersSucceed() throws Exception { + // Genuine offline / unused group: every broker answered, offset table empty. + RocketMqAgent.ensureConsumeStatsProbeSucceeded(1, 1, true, null, "GID_A"); + RocketMqAgent.ensureConsumeStatsProbeSucceeded(3, 3, true, null, "GID_A"); + } + + @Test + void ensureConsumeStatsProbeSucceededAllowsPartialFailureWhenMergeHasOffsets() throws Exception { + // One broker failed but another returned queue offsets — keep partial lag. + RocketMqAgent.ensureConsumeStatsProbeSucceeded( + 3, 1, false, new RuntimeException("partial"), "GID_A"); + } + + @Test + void shouldFailClosedOnCollisionEmptyWhenSiblingFallbacksUnreachable() { + assertTrue(RocketMqAgent.shouldFailClosedOnCollisionEmpty(true, 1, 0)); + assertTrue(RocketMqAgent.shouldFailClosedOnCollisionEmpty(true, 2, 1)); + assertFalse(RocketMqAgent.shouldFailClosedOnCollisionEmpty(false, 1, 0)); + assertFalse(RocketMqAgent.shouldFailClosedOnCollisionEmpty(true, 0, 0)); + assertFalse(RocketMqAgent.shouldFailClosedOnCollisionEmpty(true, 2, 2)); + } + + @Test + void shouldFailClosedOnCollisionPartialMutationWhenSiblingFallbacksUnreachable() { + assertTrue(RocketMqAgent.shouldFailClosedOnCollisionPartialMutation(1, 0)); + assertTrue(RocketMqAgent.shouldFailClosedOnCollisionPartialMutation(2, 1)); + assertFalse(RocketMqAgent.shouldFailClosedOnCollisionPartialMutation(0, 0)); + assertFalse(RocketMqAgent.shouldFailClosedOnCollisionPartialMutation(2, 2)); + } + + @Test + void ensureSubscriptionGroupMutationSucceededRejectsCollisionPartialUpdate() { + Exception cause = new RuntimeException("fallback unreachable"); + MQClientException err = assertThrows( + MQClientException.class, + () -> RocketMqAgent.ensureSubscriptionGroupMutationSucceeded( + 1, 1, 1, 0, cause, "delete", "GID_A")); + assertTrue(err.getMessage().contains("Docker remap address collision")); + assertSame(cause, err.getCause()); + } + + @Test + void ensureSubscriptionGroupMutationSucceededRejectsPartialRemappedUpdate() { + Exception cause = new RuntimeException("broker-b down"); + MQClientException err = assertThrows( + MQClientException.class, + () -> RocketMqAgent.ensureSubscriptionGroupMutationSucceeded( + 2, 1, 0, 0, cause, "update", "GID_A")); + assertTrue(err.getMessage().contains("partial remapped update")); + assertSame(cause, err.getCause()); + } + + @Test + void ensureSubscriptionGroupMutationSucceededAllowsFullCoverage() throws Exception { + RocketMqAgent.ensureSubscriptionGroupMutationSucceeded(2, 2, 0, 0, null, "delete", "GID_A"); + RocketMqAgent.ensureSubscriptionGroupMutationSucceeded(1, 1, 1, 1, null, "update", "GID_A"); + } + + @Test + void masterBrokerAddrsFromClusterInfoKeepsCollisionFallbacksSeparate() { + org.apache.rocketmq.remoting.protocol.body.ClusterInfo clusterInfo = + new org.apache.rocketmq.remoting.protocol.body.ClusterInfo(); + HashMap table = new HashMap<>(); + table.put("broker-a", brokerData("broker-a", "172.18.0.2:10911")); + table.put("broker-b", brokerData("broker-b", "172.18.0.3:10911")); + clusterInfo.setBrokerAddrTable(table); + + JsonObject conn = JsonParser.parseString(""" + {"namesrvAddr":"127.0.0.1:9876"} + """).getAsJsonObject(); + RocketMqAgent.MasterBrokerAddrPlan plan = + RocketMqAgent.masterBrokerAddrsFromClusterInfo(clusterInfo, conn, null); + + // Both masters remap to the same published host:port; one remapped + one fallback. + assertEquals(1, plan.remappedCount()); + assertEquals(1, plan.fallbackCount()); + assertTrue(plan.allAddrs().contains("127.0.0.1:10911")); + assertTrue(plan.isCollisionFallback("172.18.0.3:10911") || plan.isCollisionFallback("172.18.0.2:10911")); + // Remapped answered with offsets: collision fallbacks may stay unreachable on host agents. + try { + RocketMqAgent.ensureConsumeStatsProbeSucceeded( + plan.remappedCount(), 1, false, new RuntimeException("fallback unreachable"), "GID_A"); + } catch (MQClientException e) { + throw new AssertionError("non-empty remapped merge should be allowed", e); + } + // Empty merge is still allowed by the shared gate when remappedCount==success; the + // examineConsumeStatsOnMasters collision check rejects empty+unreachable siblings. + try { + RocketMqAgent.ensureConsumeStatsProbeSucceeded( + plan.remappedCount(), 1, true, null, "GID_A"); + } catch (MQClientException e) { + throw new AssertionError("empty remapped-only success remains allowed at gate layer", e); + } + } + + private static org.apache.rocketmq.remoting.protocol.route.BrokerData brokerData( + String name, String masterAddr) { + org.apache.rocketmq.remoting.protocol.route.BrokerData data = + new org.apache.rocketmq.remoting.protocol.route.BrokerData(); + data.setBrokerName(name); + HashMap addrs = new HashMap<>(); + addrs.put(0L, masterAddr); + data.setBrokerAddrs(addrs); + return data; + } } diff --git a/apps/desktop/src/components/connection/ConnectionDialog.vue b/apps/desktop/src/components/connection/ConnectionDialog.vue index eea741113..f86036607 100644 --- a/apps/desktop/src/components/connection/ConnectionDialog.vue +++ b/apps/desktop/src/components/connection/ConnectionDialog.vue @@ -2896,12 +2896,13 @@ const productionDatabaseSummary = computed(() => { return t("production.databasesSelectedCount", { selected, total: productionDatabaseNames.value.length }); }); const productionScope = computed({ - get: () => (isSingleDatabase(form.value.db_type) || form.value.is_production ? "connection" : "databases"), + get: () => (isSingleDatabase(form.value.db_type) || form.value.db_type === "mq" || form.value.db_type === "mqtt" || form.value.is_production ? "connection" : "databases"), set: (scope) => { - form.value.is_production = isSingleDatabase(form.value.db_type) || scope === "connection"; + form.value.is_production = isSingleDatabase(form.value.db_type) || form.value.db_type === "mq" || form.value.db_type === "mqtt" || scope === "connection"; }, }); -const canSelectProductionDatabases = computed(() => !isSingleDatabase(form.value.db_type)); +// MQ/MQTT have no database list — production protection is always connection-scoped. +const canSelectProductionDatabases = computed(() => !isSingleDatabase(form.value.db_type) && form.value.db_type !== "mq" && form.value.db_type !== "mqtt"); function setProductionProtectionEnabled(enabled: boolean) { productionProtectionEnabled.value = enabled; @@ -3415,8 +3416,8 @@ function connectionConfigForSubmit(id: string, generatedName = ""): ConnectionCo } if (!config.one_time) config.one_time = undefined; if (!config.read_only) config.read_only = undefined; - if (isSingleDatabase(config.db_type) && config.production_databases?.length) { - // Single-database drivers expose schemas or internal names, not independently selectable databases. + if ((isSingleDatabase(config.db_type) || config.db_type === "mq" || config.db_type === "mqtt") && config.production_databases?.length) { + // Single-database / MQ drivers expose no independently selectable database list for PROD scope. config.is_production = true; config.production_databases = []; } diff --git a/apps/desktop/src/components/mq/BrokerPanel.vue b/apps/desktop/src/components/mq/BrokerPanel.vue index 9b9b69a1c..ec13b7cf6 100644 --- a/apps/desktop/src/components/mq/BrokerPanel.vue +++ b/apps/desktop/src/components/mq/BrokerPanel.vue @@ -205,6 +205,8 @@ onUnmounted(() => { diff --git a/apps/desktop/src/components/mq/rocketmq/RocketMqTopicDialogs.vue b/apps/desktop/src/components/mq/rocketmq/RocketMqTopicDialogs.vue index 90bbd6111..3dd4c31b0 100644 --- a/apps/desktop/src/components/mq/rocketmq/RocketMqTopicDialogs.vue +++ b/apps/desktop/src/components/mq/rocketmq/RocketMqTopicDialogs.vue @@ -2,8 +2,9 @@ import { computed, ref, watch } from "vue"; import { useI18n } from "vue-i18n"; import type { BrokerNode, ResetPosition, SubscriptionInfo, TopicInfo, TopicRef } from "@/types/mq"; -import { mqAlterTopicConfig, mqGetTopicInternalStats, mqGetTopicRoute, mqGetTopicStats, mqListSubscriptions, mqResetCursor, mqSkipTopicAccumulation } from "@/lib/backend/api"; +import { mqAlterTopicConfig, mqEnrichSubscriptions, mqGetTopicInternalStats, mqGetTopicRoute, mqGetTopicStats, mqListSubscriptions, mqResetCursor, mqSkipTopicAccumulation } from "@/lib/backend/api"; import { formatError } from "@/lib/backend/errorUtils"; +import { useMqMutationGuard } from "@/composables/useMqMutationGuard"; export type RocketMqTopicDialogKind = "status" | "route" | "consumers" | "config" | "reset" | "skip"; @@ -46,6 +47,7 @@ const emit = defineEmits<{ }>(); const { t } = useI18n(); +const { confirmMqWrite } = useMqMutationGuard(() => props.connectionId); const loading = ref(false); const dialogError = ref(); @@ -194,8 +196,13 @@ async function loadResetSubscriptions(topic: TopicRef): Promise diff --git a/apps/desktop/src/components/mq/shared/mqPanel.css b/apps/desktop/src/components/mq/shared/mqPanel.css index 266080384..5ae722521 100644 --- a/apps/desktop/src/components/mq/shared/mqPanel.css +++ b/apps/desktop/src/components/mq/shared/mqPanel.css @@ -1,3 +1,6 @@ +/* Shared MQ panel chrome + button system. + * Prefer .mq-btn-* in new code; .btn-* aliases keep existing panels consistent. */ + .mq-panel-toolbar { display: flex; justify-content: space-between; @@ -51,9 +54,20 @@ font-size: 12px; } +/* ---- Buttons (canonical + legacy aliases) ---- */ + .mq-btn-primary, .mq-btn-secondary, -.mq-btn-sm { +.mq-btn-sm, +.mq-btn-danger, +.btn-primary, +.btn-secondary, +.btn-sm, +.btn-danger { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 6px; padding: 6px 12px; border: 1px solid var(--color-border); border-radius: var(--dbx-radius-fixed-6); @@ -61,35 +75,115 @@ color: var(--color-text); cursor: pointer; font-size: 13px; - transition: all 0.2s; -} - -.mq-btn-primary { - background: var(--color-primary); - border-color: var(--color-primary); - color: var(--color-primary-foreground); + font-weight: 500; + line-height: 1.2; + white-space: nowrap; + transition: + background 0.15s ease, + border-color 0.15s ease, + color 0.15s ease, + filter 0.15s ease; } +/* MQ chrome stays on the page surface in light theme — avoid solid --color-primary + * fills (often near-black) that clash with toolbar/refresh and table action buttons. */ +.mq-btn-primary:hover:not(:disabled), .mq-btn-secondary:hover:not(:disabled), -.mq-btn-sm:hover:not(:disabled) { +.mq-btn-sm:hover:not(:disabled), +.btn-primary:hover:not(:disabled), +.btn-secondary:hover:not(:disabled), +.btn-sm:hover:not(:disabled) { background: var(--color-hover); } -.mq-btn-primary:hover:not(:disabled) { - filter: brightness(1.05); +.mq-btn-danger, +.mq-btn-sm.mq-btn-danger, +.mq-btn-sm.btn-danger, +.btn-danger, +.btn-sm.btn-danger, +.btn-sm.danger { + color: var(--color-error); + border-color: var(--color-error); + background: var(--color-background); +} + +.mq-btn-danger:hover:not(:disabled), +.mq-btn-sm.mq-btn-danger:hover:not(:disabled), +.mq-btn-sm.btn-danger:hover:not(:disabled), +.btn-danger:hover:not(:disabled), +.btn-sm.btn-danger:hover:not(:disabled), +.btn-sm.danger:hover:not(:disabled) { + background: var(--color-error); + color: #fff; + filter: none; +} + +.mq-btn-sm, +.btn-sm { + padding: 4px 10px; + font-size: 12px; } .mq-btn-primary:disabled, .mq-btn-secondary:disabled, -.mq-btn-sm:disabled { +.mq-btn-sm:disabled, +.mq-btn-danger:disabled, +.btn-primary:disabled, +.btn-secondary:disabled, +.btn-sm:disabled, +.btn-danger:disabled { + opacity: 0.5; + cursor: not-allowed; + filter: none; +} + +/* Icon buttons only — do not style .btn-icon on nested SVG/icons */ +.mq-btn-icon, +button.btn-icon { + display: flex; + align-items: center; + justify-content: center; + width: 32px; + height: 32px; + padding: 0; + border: 1px solid var(--color-border); + border-radius: var(--dbx-radius-fixed-6); + background: var(--color-background); + color: var(--color-text-secondary); + cursor: pointer; + font-size: 16px; + flex-shrink: 0; + transition: + background 0.15s ease, + color 0.15s ease; +} + +.mq-btn-icon:hover:not(:disabled), +button.btn-icon:hover:not(:disabled) { + background: var(--color-background-secondary); + color: var(--color-text); +} + +.mq-btn-icon:disabled, +button.btn-icon:disabled { opacity: 0.5; cursor: not-allowed; } -.mq-btn-sm.btn-danger, -.mq-btn-sm.mq-btn-danger { - color: var(--color-error); - border-color: var(--color-error); +.mq-btn-close, +.btn-close { + border: none; + background: none; + padding: 0; + font-size: 22px; + line-height: 1; + cursor: pointer; + color: var(--color-text-secondary); +} + +.mq-btn-close:hover, +.btn-close:hover { + color: var(--color-text); } .mq-query-mode-tabs { @@ -154,26 +248,6 @@ background: var(--color-primary-alpha); } -.mq-btn-icon { - display: flex; - align-items: center; - justify-content: center; - width: 32px; - height: 32px; - border: 1px solid var(--color-border); - border-radius: var(--dbx-radius-fixed-6); - background: var(--color-background); - color: var(--color-text-secondary); - cursor: pointer; - font-size: 16px; - flex-shrink: 0; -} - -.mq-btn-icon:disabled { - opacity: 0.5; - cursor: not-allowed; -} - .mq-spin { display: inline-block; animation: mq-spin-anim 0.8s linear infinite; diff --git a/apps/desktop/src/composables/__tests__/useMqMutationGuard.spec.ts b/apps/desktop/src/composables/__tests__/useMqMutationGuard.spec.ts new file mode 100644 index 000000000..3207845e4 --- /dev/null +++ b/apps/desktop/src/composables/__tests__/useMqMutationGuard.spec.ts @@ -0,0 +1,75 @@ +// @vitest-environment happy-dom + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createPinia, setActivePinia } from "pinia"; + +const requestConfirmation = vi.fn(); +const toast = vi.fn(); + +vi.mock("@/stores/connectionStore", () => ({ + useConnectionStore: () => ({ + getConfig: (id: string) => { + if (id === "missing") return undefined; + if (id === "readonly") return { name: "ro", read_only: true, is_production: false }; + if (id === "prod") return { name: "prod-mq", read_only: false, is_production: true }; + return { name: "dev-mq", read_only: false, is_production: false }; + }, + }), +})); + +vi.mock("@/stores/productionSafetyStore", () => ({ + useProductionSafetyStore: () => ({ requestConfirmation }), +})); + +vi.mock("@/composables/useToast", () => ({ + useToast: () => ({ toast }), +})); + +vi.mock("vue-i18n", () => ({ + useI18n: () => ({ + t: (key: string) => key, + }), +})); + +import { useMqMutationGuard } from "@/composables/useMqMutationGuard"; + +describe("useMqMutationGuard", () => { + beforeEach(() => { + setActivePinia(createPinia()); + requestConfirmation.mockReset(); + requestConfirmation.mockResolvedValue(true); + toast.mockReset(); + }); + + it("denies missing and read-only connections without prompting", async () => { + const missing = useMqMutationGuard("missing"); + const readonly = useMqMutationGuard("readonly"); + await expect(missing.confirmMqWrite("send")).resolves.toBe(false); + await expect(readonly.confirmMqWrite("send")).resolves.toBe(false); + expect(requestConfirmation).not.toHaveBeenCalled(); + expect(toast).toHaveBeenCalledWith("mqAdmin.connectionMissing"); + expect(toast).toHaveBeenCalledWith("mqAdmin.writeDeniedReadOnly"); + }); + + it("allows non-production writes immediately", async () => { + const guard = useMqMutationGuard("dev"); + await expect(guard.confirmMqWrite("send")).resolves.toBe(true); + expect(requestConfirmation).not.toHaveBeenCalled(); + }); + + it("resolves getter connection ids used by MQ panels", async () => { + const guard = useMqMutationGuard(() => "dev"); + await expect(guard.confirmMqWrite("send")).resolves.toBe(true); + expect(toast).not.toHaveBeenCalledWith("mqAdmin.connectionMissing"); + }); + + it("prompts for production writes", async () => { + const guard = useMqMutationGuard("prod"); + await expect(guard.confirmMqWrite("send")).resolves.toBe(true); + expect(requestConfirmation).toHaveBeenCalledWith({ + sql: "send", + connectionName: "prod-mq", + source: "production.sourceMq", + }); + }); +}); diff --git a/apps/desktop/src/composables/useMqMutationGuard.ts b/apps/desktop/src/composables/useMqMutationGuard.ts new file mode 100644 index 000000000..6a0338e77 --- /dev/null +++ b/apps/desktop/src/composables/useMqMutationGuard.ts @@ -0,0 +1,40 @@ +import { toValue, type MaybeRefOrGetter } from "vue"; +import { useI18n } from "vue-i18n"; +import { useConnectionStore } from "@/stores/connectionStore"; +import { useProductionSafetyStore } from "@/stores/productionSafetyStore"; +import { useToast } from "@/composables/useToast"; + +/** + * Shared write guard for MQ console mutations. + * Read-only is enforced by the backend; production connections require an + * explicit confirmation (same dialog as SQL production writes). + */ +export function useMqMutationGuard(connectionId: MaybeRefOrGetter) { + const connectionStore = useConnectionStore(); + const productionSafetyStore = useProductionSafetyStore(); + const { toast } = useToast(); + const { t } = useI18n(); + + async function confirmMqWrite(operation: string): Promise { + // Call sites pass () => props.connectionId — must use toValue (not unref) + // so getters resolve to the real connection id. + const id = toValue(connectionId); + const config = connectionStore.getConfig(id); + if (!config) { + toast(t("mqAdmin.connectionMissing")); + return false; + } + if (config.read_only) { + toast(t("mqAdmin.writeDeniedReadOnly")); + return false; + } + if (!config.is_production) return true; + return productionSafetyStore.requestConfirmation({ + sql: operation, + connectionName: config.name, + source: t("production.sourceMq"), + }); + } + + return { confirmMqWrite }; +} diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts index 7089c5185..3115eaf15 100644 --- a/apps/desktop/src/i18n/locales/en.ts +++ b/apps/desktop/src/i18n/locales/en.ts @@ -3991,6 +3991,7 @@ export default { sourceDataCompare: "Data compare", sourceExtension: "Extension manager", sourceSidebar: "Object tree", + sourceMq: "Message queue", sourceObjectSource: "Object source editor", sourceDataGenerate: "Data generator", sourceQueryHistory: "Query history rollback", @@ -6061,6 +6062,8 @@ export default { viewTenant: "View tenant", viewNamespace: "View namespace", viewTopic: "View topic", + connectionMissing: "Connection not found. Reconnect and try again.", + writeDeniedReadOnly: "This connection is read-only and cannot perform write operations.", readOnly: "Read-only", allNamespaces: "All namespaces", newNamespace: "New namespace", @@ -6352,8 +6355,13 @@ export default { loading: "Loading...", refresh: "Refresh", refreshing: "Refreshing...", + enriching: "Loading online details...", + truncatedHint: "Showing the first {count} consumer groups (list capped). Narrow the search to find others.", + enrichFailedHint: "Online member/topic details could not be loaded ({error}). Showing the fast list only.", noSubscriptions: "No subscriptions for this topic", noConsumerGroups: "No consumer groups", + noMatches: "No matching consumer groups", + backlogPartialFailed: "Failed to load backlog for {count} topic(s): {error}", viewDetail: "Details", editConfig: "Config", save: "Save", @@ -6364,7 +6372,18 @@ export default { consumerConsumeDetail: "Consumption details", noOnlineConsumers: "No online consumers", noConsumeDetail: "No consumption details", + noQueueConsumeProgress: "No queue consumption progress", operationTopic: "Topic", + consumeDelay: "Backlog", + invalidTopicScope: "Invalid topic scope", + lastConsumeTime: "Last consumption time", + broker: "broker", + queue: "queue", + consumerClient: "consumerClient", + brokerOffset: "brokerOffset", + consumerOffset: "consumerOffset", + diffTotal: "diffTotal", + lastTimestamp: "lastTimestamp", consumeEnable: "Consume enabled", consumeBroadcastEnable: "Broadcast consume enabled", consumeFromMinEnable: "Consume from min offset", @@ -6381,6 +6400,7 @@ export default { normal: "Normal", fifo: "Ordered", system: "System", + unknown: "Unknown", }, rocketmqGroupMode: { clustering: "Cluster", @@ -6624,6 +6644,29 @@ export default { searchHint: "Search for messages first, then click View trace", viewTrace: "View trace", detailTitle: "Message trace details", + typePub: "Pub", + typeSubBefore: "SubBefore", + typeSubAfter: "SubAfter", + typeEndTransaction: "EndTransaction", + typeUnknown: "Unknown", + statusSuccess: "Success", + statusFail: "Failed", + fieldRegionId: "Region", + fieldGroup: "Group", + fieldTopic: "Topic", + fieldMsgId: "MsgId", + fieldTags: "Tags", + fieldKeys: "Keys", + fieldStoreHost: "Store host", + fieldClientHost: "Client host", + fieldBodyLength: "Body length", + fieldCostTime: "Cost", + fieldMsgType: "Msg type", + fieldOffsetMsgId: "Offset MsgId", + fieldRequestId: "Request ID", + fieldRetryTimes: "Retry", + fieldContextCode: "Context code", + rawPayload: "Raw trace", traceTopicRouteMissing: "The trace topic has no route on NameServer (CODE: 17).\n\nCommon causes:\n1. Broker trace is disabled: set traceTopicEnable=true in broker.conf and restart the broker\n2. Producer/Consumer trace is disabled: call setEnableTrace(true)\n3. A custom trace topic is used: select the correct trace topic above", }, diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts index 837409e29..6cb38c8cb 100644 --- a/apps/desktop/src/i18n/locales/es.ts +++ b/apps/desktop/src/i18n/locales/es.ts @@ -5917,6 +5917,8 @@ export default withEnglishFallback({ viewTenant: "Ver tenant", viewNamespace: "Ver namespace", viewTopic: "Ver tema", + connectionMissing: "No se encontró la conexión. Vuelve a conectar e inténtalo de nuevo.", + writeDeniedReadOnly: "Esta conexión es de solo lectura y no puede realizar operaciones de escritura.", readOnly: "Solo lectura", allNamespaces: "Todos los namespaces", newNamespace: "Nuevo namespace", @@ -6258,7 +6260,12 @@ export default withEnglishFallback({ searchPlaceholder: "Buscar nombre de suscripción", subscribedTopics: "Topics suscritos", refreshing: "Actualizando...", + enriching: "Cargando detalles en línea...", + truncatedHint: "Mostrando los primeros {count} grupos de consumidores (límite). Restrinja la búsqueda para encontrar otros.", + enrichFailedHint: "No se pudieron cargar los detalles en línea ({error}). Solo se muestra la lista rápida.", noConsumerGroups: "No hay grupos de consumidores", + noMatches: "No hay grupos de consumidores coincidentes", + backlogPartialFailed: "Error al cargar el backlog de {count} topic(s): {error}", viewDetail: "Detalles", editConfig: "Configurar", save: "Guardar", @@ -6269,7 +6276,18 @@ export default withEnglishFallback({ consumerConsumeDetail: "Detalles de consumo", noOnlineConsumers: "No hay consumidores en línea actualmente", noConsumeDetail: "Sin detalles de consumo", + noQueueConsumeProgress: "Sin progreso de consumo por cola", operationTopic: "Topic", + consumeDelay: "Backlog", + invalidTopicScope: "Ámbito de tema no válido", + lastConsumeTime: "Último consumo", + broker: "broker", + queue: "queue", + consumerClient: "consumerClient", + brokerOffset: "brokerOffset", + consumerOffset: "consumerOffset", + diffTotal: "diffTotal", + lastTimestamp: "lastTimestamp", consumeEnable: "Permitir consumo", consumeBroadcastEnable: "Permitir consumo en difusión", consumeFromMinEnable: "Comenzar desde el offset mínimo", @@ -6284,6 +6302,7 @@ export default withEnglishFallback({ normal: "Normal", fifo: "FIFO", system: "Sistema", + unknown: "Desconocido", }, rocketmqGroupMode: { clustering: "Cluster", @@ -6522,6 +6541,7 @@ export default withEnglishFallback({ sourceDataCompare: "Comparación de datos", sourceExtension: "Gestor de extensiones", sourceSidebar: "Árbol de objetos", + sourceMq: "Cola de mensajes", sourceObjectSource: "Editor de código fuente de objetos", sourceDataGenerate: "Generador de datos", sourceQueryHistory: "Reversión del historial de consultas", @@ -6916,5 +6936,28 @@ export default withEnglishFallback({ detailTitle: "Detalles de la traza de consumo", traceTopicRouteMissing: "El Topic de trazas no tiene información de ruta en NameServer (CODE: 17).\n\nCausas comunes:\n1. Broker no tiene habilitada la traza de mensajes: configure traceTopicEnable=true en broker.conf y reinicie el Broker\n2. Producer/Consumer no tiene habilitada la traza: setEnableTrace(true)\n3. Se utilizó un Topic de trazas personalizado, seleccione el Topic de trazas correcto arriba.", + typePub: "Pub", + typeSubBefore: "SubBefore", + typeSubAfter: "SubAfter", + typeEndTransaction: "EndTransaction", + typeUnknown: "Desconocido", + statusSuccess: "Correcto", + statusFail: "Fallido", + fieldRegionId: "Region", + fieldGroup: "Group", + fieldTopic: "Topic", + fieldMsgId: "MsgId", + fieldTags: "Tags", + fieldKeys: "Keys", + fieldStoreHost: "Store host", + fieldClientHost: "Client host", + fieldBodyLength: "Body length", + fieldCostTime: "Coste", + fieldMsgType: "Msg type", + fieldOffsetMsgId: "Offset MsgId", + fieldRequestId: "Request ID", + fieldRetryTimes: "Reintentos", + fieldContextCode: "Context code", + rawPayload: "Traza en bruto", }, }); diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts index a4e915895..010429615 100644 --- a/apps/desktop/src/i18n/locales/it.ts +++ b/apps/desktop/src/i18n/locales/it.ts @@ -5917,6 +5917,8 @@ export default withEnglishFallback({ viewTenant: "Visualizza tenant", viewNamespace: "Visualizza namespace", viewTopic: "Visualizza topic", + connectionMissing: "Connessione non trovata. Riconnetti e riprova.", + writeDeniedReadOnly: "La connessione corrente è in modalità sola lettura, impossibile eseguire operazioni di scrittura.", readOnly: "Sola lettura", allNamespaces: "Tutti i namespace", newNamespace: "Nuovo namespace", @@ -6258,7 +6260,12 @@ export default withEnglishFallback({ searchPlaceholder: "Cerca nome sottoscrizione", subscribedTopics: "Topic sottoscritti", refreshing: "Aggiornamento in corso...", + enriching: "Caricamento dettagli online...", + truncatedHint: "Vengono mostrati i primi {count} consumer group (limite). Restringi la ricerca per trovarne altri.", + enrichFailedHint: "Impossibile caricare i dettagli online ({error}). Viene mostrata solo l'elenco rapido.", noConsumerGroups: "Nessun Consumer Group", + noMatches: "Nessun consumer group corrispondente", + backlogPartialFailed: "Caricamento backlog non riuscito per {count} topic: {error}", viewDetail: "Dettagli", editConfig: "Configurazione", save: "Salva", @@ -6269,7 +6276,18 @@ export default withEnglishFallback({ consumerConsumeDetail: "Dettagli consumo", noOnlineConsumers: "Nessun consumatore online", noConsumeDetail: "Nessun dettaglio consumo", + noQueueConsumeProgress: "Nessun progresso di consumo per coda", operationTopic: "Topic", + consumeDelay: "Backlog", + invalidTopicScope: "Ambito topic non valido", + lastConsumeTime: "Ultimo consumo", + broker: "broker", + queue: "queue", + consumerClient: "consumerClient", + brokerOffset: "brokerOffset", + consumerOffset: "consumerOffset", + diffTotal: "diffTotal", + lastTimestamp: "lastTimestamp", consumeEnable: "Abilita consumo", consumeBroadcastEnable: "Abilita consumo broadcast", consumeFromMinEnable: "Inizia dall'offset minimo", @@ -6284,6 +6302,7 @@ export default withEnglishFallback({ normal: "Normale", fifo: "FIFO", system: "Sistema", + unknown: "Sconosciuto", }, rocketmqGroupMode: { clustering: "Cluster", @@ -6522,6 +6541,7 @@ export default withEnglishFallback({ sourceDataCompare: "Confronto dati", sourceExtension: "Gestione estensioni", sourceSidebar: "Albero oggetti", + sourceMq: "Coda di messaggi", sourceObjectSource: "Editor sorgente oggetto", sourceDataGenerate: "Generatore dati", sourceQueryHistory: "Rollback cronologia query", @@ -6916,5 +6936,28 @@ export default withEnglishFallback({ detailTitle: "Dettagli traccia consumo", traceTopicRouteMissing: "Il Topic di traccia non ha informazioni di instradamento sul NameServer (CODE: 17).\n\nCause comuni:\n1. Il Broker non ha abilitato la traccia messaggi: imposta traceTopicEnable=true in broker.conf e riavvia il Broker\n2. Producer/Consumer non hanno abilitato la traccia: setEnableTrace(true)\n3. È stato utilizzato un Topic di traccia personalizzato, seleziona il Topic di traccia corretto sopra.", + typePub: "Pub", + typeSubBefore: "SubBefore", + typeSubAfter: "SubAfter", + typeEndTransaction: "EndTransaction", + typeUnknown: "Sconosciuto", + statusSuccess: "Riuscito", + statusFail: "Non riuscito", + fieldRegionId: "Region", + fieldGroup: "Group", + fieldTopic: "Topic", + fieldMsgId: "MsgId", + fieldTags: "Tags", + fieldKeys: "Keys", + fieldStoreHost: "Store host", + fieldClientHost: "Client host", + fieldBodyLength: "Body length", + fieldCostTime: "Costo", + fieldMsgType: "Msg type", + fieldOffsetMsgId: "Offset MsgId", + fieldRequestId: "Request ID", + fieldRetryTimes: "Tentativi", + fieldContextCode: "Context code", + rawPayload: "Traccia grezza", }, }); diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts index de0e8fdfb..0577dcad5 100644 --- a/apps/desktop/src/i18n/locales/ja.ts +++ b/apps/desktop/src/i18n/locales/ja.ts @@ -5972,6 +5972,8 @@ export default withEnglishFallback({ viewTenant: "テナントを表示", viewNamespace: "名前空間を表示", viewTopic: "トピックを表示", + connectionMissing: "接続が見つかりません。再接続してから再試行してください。", + writeDeniedReadOnly: "現在の接続は読み取り専用モードです。書き込み操作は実行できません。", readOnly: "読み取り専用", allNamespaces: "すべての名前空間", newNamespace: "新規名前空間", @@ -6313,7 +6315,12 @@ export default withEnglishFallback({ searchPlaceholder: "サブスクリプション名を検索", subscribedTopics: "サブスクライブされた Topic", refreshing: "更新中...", + enriching: "オンライン詳細を読み込み中...", + truncatedHint: "最初の {count} 件のコンシューマグループのみ表示しています（上限）。検索条件を絞ってください。", + enrichFailedHint: "オンラインメンバー/購読 Topic の読み込みに失敗しました（{error}）。高速一覧のみ表示しています。", noConsumerGroups: "Consumer Group がありません", + noMatches: "一致するコンシューマグループがありません", + backlogPartialFailed: "{count} 件の Topic のバックログ読み込みに失敗しました: {error}", viewDetail: "詳細", editConfig: "設定", save: "保存", @@ -6324,7 +6331,18 @@ export default withEnglishFallback({ consumerConsumeDetail: "消費詳細", noOnlineConsumers: "現在オンラインのコンシューマはいません", noConsumeDetail: "消費詳細がありません", + noQueueConsumeProgress: "キューの消費進捗がありません", operationTopic: "Topic", + consumeDelay: "バックログ", + invalidTopicScope: "トピックのスコープが無効です", + lastConsumeTime: "最終消費時間", + broker: "broker", + queue: "queue", + consumerClient: "consumerClient", + brokerOffset: "brokerOffset", + consumerOffset: "consumerOffset", + diffTotal: "diffTotal", + lastTimestamp: "lastTimestamp", consumeEnable: "消費を許可", consumeBroadcastEnable: "ブロードキャスト消費を許可", consumeFromMinEnable: "最小 offset から開始", @@ -6339,6 +6357,7 @@ export default withEnglishFallback({ normal: "通常", fifo: "FIFO", system: "システム", + unknown: "不明", }, rocketmqGroupMode: { clustering: "クラスタリング", @@ -6577,6 +6596,7 @@ export default withEnglishFallback({ sourceDataCompare: "データ比較", sourceExtension: "拡張機能マネージャー", sourceSidebar: "オブジェクトツリー", + sourceMq: "メッセージキュー", sourceObjectSource: "オブジェクトソースエディター", sourceDataGenerate: "データジェネレーター", sourceQueryHistory: "クエリ履歴のロールバック", @@ -6971,5 +6991,28 @@ export default withEnglishFallback({ detailTitle: "消費トレース詳細", traceTopicRouteMissing: "トレース Topic が NameServer にルート情報がありません (CODE: 17)。\n\n一般的な原因:\n1. Broker でメッセージトレースが有効になっていない: broker.conf で traceTopicEnable=true を設定し Broker を再起動\n2. Producer/Consumer でトレースが有効になっていない: setEnableTrace(true)\n3. カスタムトレース Topic を使用している場合は、上で正しいトレース Topic を選択してください", + typePub: "Pub", + typeSubBefore: "SubBefore", + typeSubAfter: "SubAfter", + typeEndTransaction: "EndTransaction", + typeUnknown: "不明", + statusSuccess: "成功", + statusFail: "失敗", + fieldRegionId: "Region", + fieldGroup: "Group", + fieldTopic: "Topic", + fieldMsgId: "MsgId", + fieldTags: "Tags", + fieldKeys: "Keys", + fieldStoreHost: "Store Host", + fieldClientHost: "Client Host", + fieldBodyLength: "Body Length", + fieldCostTime: "所要時間", + fieldMsgType: "メッセージ種別", + fieldOffsetMsgId: "Offset MsgId", + fieldRequestId: "Request ID", + fieldRetryTimes: "リトライ回数", + fieldContextCode: "Context Code", + rawPayload: "生トレース", }, }); diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts index bf6f1e38a..bd8485b96 100644 --- a/apps/desktop/src/i18n/locales/ko.ts +++ b/apps/desktop/src/i18n/locales/ko.ts @@ -3538,6 +3538,7 @@ export default withEnglishFallback({ sourceDataCompare: "데이터 비교", sourceExtension: "확장 관리자", sourceSidebar: "객체 트리", + sourceMq: "메시지 큐", sourceObjectSource: "객체 소스 편집기", sourceDataGenerate: "데이터 생성기", sourceQueryHistory: "쿼리 기록 롤백", @@ -5585,6 +5586,8 @@ export default withEnglishFallback({ viewTenant: "테넌트 보기", viewNamespace: "네임스페이스 보기", viewTopic: "토픽 보기", + connectionMissing: "연결을 찾을 수 없습니다. 다시 연결한 후 시도하세요.", + writeDeniedReadOnly: "이 연결은 읽기 전용이며 쓰기 작업을 수행할 수 없습니다.", readOnly: "읽기 전용", allNamespaces: "모든 네임스페이스", newNamespace: "새 네임스페이스", @@ -5876,8 +5879,13 @@ export default withEnglishFallback({ loading: "불러오는 중...", refresh: "새로고침", refreshing: "새로고침 중...", + enriching: "온라인 상세 불러오는 중...", + truncatedHint: "처음 {count}개의 컨슈머 그룹만 표시합니다(상한). 검색 범위를 좁혀 주세요.", + enrichFailedHint: "온라인 멤버/구독 토픽 정보를 불러오지 못했습니다({error}). 빠른 목록만 표시합니다.", noSubscriptions: "이 토픽에 구독이 없습니다", noConsumerGroups: "컨슈머 그룹이 없습니다", + noMatches: "일치하는 컨슈머 그룹이 없습니다", + backlogPartialFailed: "{count}개 토픽의 백로그를 불러오지 못했습니다: {error}", viewDetail: "상세", editConfig: "구성", save: "저장", @@ -5888,7 +5896,18 @@ export default withEnglishFallback({ consumerConsumeDetail: "소비 상세", noOnlineConsumers: "온라인 컨슈머가 없습니다", noConsumeDetail: "소비 상세가 없습니다", + noQueueConsumeProgress: "큐 소비 진행 정보가 없습니다", operationTopic: "토픽", + consumeDelay: "적체", + invalidTopicScope: "잘못된 토픽 범위", + lastConsumeTime: "마지막 소비 시간", + broker: "broker", + queue: "queue", + consumerClient: "consumerClient", + brokerOffset: "brokerOffset", + consumerOffset: "consumerOffset", + diffTotal: "diffTotal", + lastTimestamp: "lastTimestamp", consumeEnable: "소비 활성화", consumeBroadcastEnable: "브로드캐스트 소비 활성화", consumeFromMinEnable: "최소 오프셋부터 소비", @@ -5905,6 +5924,7 @@ export default withEnglishFallback({ normal: "일반", fifo: "순서 보장", system: "시스템", + unknown: "알 수 없음", }, rocketmqGroupMode: { clustering: "클러스터", @@ -6149,6 +6169,29 @@ export default withEnglishFallback({ detailTitle: "메시지 추적 상세", traceTopicRouteMissing: "추적 토픽이 NameServer에 라우트 정보가 없습니다 (CODE: 17).\n\n일반적인 원인:\n1. Broker 추적이 비활성화됨: broker.conf에서 traceTopicEnable=true를 설정하고 Broker를 다시 시작\n2. Producer/Consumer 추적이 비활성화됨: setEnableTrace(true) 호출\n3. 사용자 지정 추적 토픽 사용: 위에서 올바른 추적 토픽을 선택하세요", + typePub: "Pub", + typeSubBefore: "SubBefore", + typeSubAfter: "SubAfter", + typeEndTransaction: "EndTransaction", + typeUnknown: "알 수 없음", + statusSuccess: "성공", + statusFail: "실패", + fieldRegionId: "Region", + fieldGroup: "Group", + fieldTopic: "Topic", + fieldMsgId: "MsgId", + fieldTags: "Tags", + fieldKeys: "Keys", + fieldStoreHost: "Store Host", + fieldClientHost: "Client Host", + fieldBodyLength: "Body Length", + fieldCostTime: "소요 시간", + fieldMsgType: "메시지 유형", + fieldOffsetMsgId: "Offset MsgId", + fieldRequestId: "Request ID", + fieldRetryTimes: "재시도", + fieldContextCode: "Context Code", + rawPayload: "원본 추적", }, mqRaw: { title: "원시 API", diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts index 26f998599..f382e769a 100644 --- a/apps/desktop/src/i18n/locales/pt-BR.ts +++ b/apps/desktop/src/i18n/locales/pt-BR.ts @@ -5919,6 +5919,8 @@ export default withEnglishFallback({ viewTenant: "Ver locatário", viewNamespace: "Ver namespace", viewTopic: "Ver tópico", + connectionMissing: "Conexão não encontrada. Reconecte e tente novamente.", + writeDeniedReadOnly: "Esta conexão é somente leitura e não pode executar operações de gravação.", readOnly: "Somente leitura", allNamespaces: "Todos os namespaces", newNamespace: "Novo namespace", @@ -6260,7 +6262,12 @@ export default withEnglishFallback({ searchPlaceholder: "Pesquisar nome da assinatura", subscribedTopics: "Tópicos inscritos", refreshing: "Atualizando...", + enriching: "Carregando detalhes online...", + truncatedHint: "Mostrando os primeiros {count} grupos de consumidores (limite). Restrinja a busca para encontrar outros.", + enrichFailedHint: "Não foi possível carregar os detalhes online ({error}). Mostrando apenas a lista rápida.", noConsumerGroups: "Nenhum Consumer Group", + noMatches: "Nenhum grupo de consumidores correspondente", + backlogPartialFailed: "Falha ao carregar backlog de {count} tópico(s): {error}", viewDetail: "Detalhes", editConfig: "Configuração", save: "Salvar", @@ -6271,7 +6278,18 @@ export default withEnglishFallback({ consumerConsumeDetail: "Detalhes do consumo", noOnlineConsumers: "Nenhum consumidor online no momento", noConsumeDetail: "Nenhum detalhe de consumo", + noQueueConsumeProgress: "Sem progresso de consumo por fila", operationTopic: "Topic", + consumeDelay: "Backlog", + invalidTopicScope: "Escopo de tópico inválido", + lastConsumeTime: "Último consumo", + broker: "broker", + queue: "queue", + consumerClient: "consumerClient", + brokerOffset: "brokerOffset", + consumerOffset: "consumerOffset", + diffTotal: "diffTotal", + lastTimestamp: "lastTimestamp", consumeEnable: "Permitir consumo", consumeBroadcastEnable: "Permitir consumo em broadcast", consumeFromMinEnable: "Iniciar do offset mínimo", @@ -6286,6 +6304,7 @@ export default withEnglishFallback({ normal: "Normal", fifo: "Ordenado", system: "Sistema", + unknown: "Desconhecido", }, rocketmqGroupMode: { clustering: "Cluster", @@ -6524,6 +6543,7 @@ export default withEnglishFallback({ sourceDataCompare: "Comparação de dados", sourceExtension: "Gerenciador de extensões", sourceSidebar: "Árvore de objetos", + sourceMq: "Fila de mensagens", sourceObjectSource: "Editor de código-fonte de objetos", sourceDataGenerate: "Gerador de dados", sourceQueryHistory: "Reversão do histórico de consultas", @@ -6918,5 +6938,28 @@ export default withEnglishFallback({ detailTitle: "Detalhes do rastreamento de consumo", traceTopicRouteMissing: "O tópico de rastreamento não possui informações de rota no NameServer (CODE: 17).\n\nCausas comuns:\n1. Broker não habilitou rastreamento de mensagens: defina traceTopicEnable=true no broker.conf e reinicie o Broker\n2. Producer/Consumer não habilitou rastreamento: setEnableTrace(true)\n3. Usou um tópico de rastreamento personalizado, selecione o tópico correto acima.", + typePub: "Pub", + typeSubBefore: "SubBefore", + typeSubAfter: "SubAfter", + typeEndTransaction: "EndTransaction", + typeUnknown: "Desconhecido", + statusSuccess: "Sucesso", + statusFail: "Falhou", + fieldRegionId: "Region", + fieldGroup: "Group", + fieldTopic: "Topic", + fieldMsgId: "MsgId", + fieldTags: "Tags", + fieldKeys: "Keys", + fieldStoreHost: "Store host", + fieldClientHost: "Client host", + fieldBodyLength: "Body length", + fieldCostTime: "Custo", + fieldMsgType: "Msg type", + fieldOffsetMsgId: "Offset MsgId", + fieldRequestId: "Request ID", + fieldRetryTimes: "Tentativas", + fieldContextCode: "Context code", + rawPayload: "Rastreamento bruto", }, }); diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts index 5373d4929..6b7ceddad 100644 --- a/apps/desktop/src/i18n/locales/zh-CN.ts +++ b/apps/desktop/src/i18n/locales/zh-CN.ts @@ -3990,6 +3990,7 @@ export default withEnglishFallback({ sourceDataCompare: "数据对比", sourceExtension: "扩展管理", sourceSidebar: "对象树", + sourceMq: "消息队列", sourceObjectSource: "对象源码编辑器", sourceDataGenerate: "数据生成器", sourceQueryHistory: "查询历史回滚", @@ -6058,6 +6059,8 @@ export default withEnglishFallback({ viewTenant: "查看租户", viewNamespace: "查看命名空间", viewTopic: "查看主题", + connectionMissing: "未找到连接，请重新连接后再试。", + writeDeniedReadOnly: "当前连接为只读模式，不能执行写操作。", readOnly: "只读", allNamespaces: "全部命名空间", newNamespace: "新建命名空间", @@ -6344,6 +6347,9 @@ export default withEnglishFallback({ createSubscription: "创建订阅", selectTopicFirst: "请先选择一个主题", searchPlaceholder: "搜索订阅名称", + enriching: "正在补全在线信息...", + truncatedHint: "当前仅显示前 {count} 个消费组（已达上限），请缩小搜索范围查找其他组", + enrichFailedHint: "在线成员/订阅 Topic 信息加载失败（{error}），当前仅显示快速列表", subscribedTopics: "订阅 Topic", readOnly: "当前连接为只读模式，不能执行写操作", loading: "加载中...", @@ -6351,6 +6357,8 @@ export default withEnglishFallback({ refreshing: "刷新中...", noSubscriptions: "该主题暂无订阅", noConsumerGroups: "暂无 Consumer Group", + noMatches: "没有匹配的 Consumer Group", + backlogPartialFailed: "{count} 个 Topic 的堆积信息加载失败：{error}", viewDetail: "详情", editConfig: "配置", save: "保存", @@ -6361,7 +6369,18 @@ export default withEnglishFallback({ consumerConsumeDetail: "消费详情", noOnlineConsumers: "当前没有在线消费者", noConsumeDetail: "暂无消费详情", + noQueueConsumeProgress: "暂无队列消费进度", operationTopic: "Topic", + consumeDelay: "积压", + invalidTopicScope: "无效的 Topic 作用域", + lastConsumeTime: "最后消费时间", + broker: "broker", + queue: "queue", + consumerClient: "consumerClient", + brokerOffset: "brokerOffset", + consumerOffset: "consumerOffset", + diffTotal: "diffTotal", + lastTimestamp: "lastTimestamp", consumeEnable: "允许消费", consumeBroadcastEnable: "允许广播消费", consumeFromMinEnable: "从最小 offset 开始", @@ -6378,6 +6397,7 @@ export default withEnglishFallback({ normal: "普通", fifo: "顺序", system: "系统", + unknown: "未知", }, rocketmqGroupMode: { clustering: "集群", @@ -6620,6 +6640,29 @@ export default withEnglishFallback({ searchHint: "请先搜索消息，再点击「查看消费轨迹」", viewTrace: "查看消费轨迹", detailTitle: "消费轨迹详情", + typePub: "Pub", + typeSubBefore: "SubBefore", + typeSubAfter: "SubAfter", + typeEndTransaction: "EndTransaction", + typeUnknown: "未知", + statusSuccess: "成功", + statusFail: "失败", + fieldRegionId: "Region", + fieldGroup: "Group", + fieldTopic: "Topic", + fieldMsgId: "MsgId", + fieldTags: "Tags", + fieldKeys: "Keys", + fieldStoreHost: "Store Host", + fieldClientHost: "Client Host", + fieldBodyLength: "Body Length", + fieldCostTime: "耗时", + fieldMsgType: "消息类型", + fieldOffsetMsgId: "Offset MsgId", + fieldRequestId: "Request ID", + fieldRetryTimes: "重试次数", + fieldContextCode: "Context Code", + rawPayload: "原始轨迹", traceTopicRouteMissing: "轨迹 Topic 在 NameServer 上没有路由信息（CODE: 17）。\n\n常见原因：\n1. Broker 未开启消息轨迹：在 broker.conf 中设置 traceTopicEnable=true 并重启 Broker\n2. Producer/Consumer 未开启轨迹：setEnableTrace(true)\n3. 使用了自定义轨迹 Topic，请在上方选择正确的轨迹 Topic", }, mqRaw: { diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts index 7ee155a4b..5af3103ec 100644 --- a/apps/desktop/src/i18n/locales/zh-TW.ts +++ b/apps/desktop/src/i18n/locales/zh-TW.ts @@ -5912,6 +5912,8 @@ export default withEnglishFallback({ viewTenant: "檢視租用戶", viewNamespace: "檢視命名空間", viewTopic: "檢視主題", + connectionMissing: "找不到連線，請重新連線後再試。", + writeDeniedReadOnly: "目前連線為唯讀模式，無法執行寫入操作。", readOnly: "唯讀", allNamespaces: "全部命名空間", newNamespace: "新建命名空間", @@ -6253,7 +6255,12 @@ export default withEnglishFallback({ searchPlaceholder: "搜尋訂閱名稱", subscribedTopics: "訂閱 Topic", refreshing: "重新整理中...", + enriching: "正在載入線上詳情...", + truncatedHint: "目前僅顯示前 {count} 個消費組（已達上限），請縮小搜尋範圍查找其他組", + enrichFailedHint: "線上成員/訂閱 Topic 資訊載入失敗（{error}），目前僅顯示快速列表", noConsumerGroups: "暫無 Consumer Group", + noMatches: "沒有符合的消費組", + backlogPartialFailed: "{count} 個 Topic 的堆積資訊載入失敗：{error}", viewDetail: "詳情", editConfig: "配置", save: "儲存", @@ -6264,7 +6271,18 @@ export default withEnglishFallback({ consumerConsumeDetail: "消費詳情", noOnlineConsumers: "當前沒有線上消費者", noConsumeDetail: "暫無消費詳情", + noQueueConsumeProgress: "暫無佇列消費進度", operationTopic: "Topic", + consumeDelay: "積壓", + invalidTopicScope: "無效的主題作用域", + lastConsumeTime: "最後消費時間", + broker: "broker", + queue: "queue", + consumerClient: "consumerClient", + brokerOffset: "brokerOffset", + consumerOffset: "consumerOffset", + diffTotal: "diffTotal", + lastTimestamp: "lastTimestamp", consumeEnable: "允許消費", consumeBroadcastEnable: "允許廣播消費", consumeFromMinEnable: "從最小 offset 開始", @@ -6279,6 +6297,7 @@ export default withEnglishFallback({ normal: "普通", fifo: "順序", system: "系統", + unknown: "未知", }, rocketmqGroupMode: { clustering: "集群", @@ -6517,6 +6536,7 @@ export default withEnglishFallback({ sourceDataCompare: "資料比對", sourceExtension: "擴充功能管理", sourceSidebar: "物件樹", + sourceMq: "訊息佇列", sourceObjectSource: "物件原始碼編輯器", sourceDataGenerate: "資料產生器", sourceQueryHistory: "查詢歷史回復", @@ -6909,6 +6929,29 @@ export default withEnglishFallback({ searchHint: "請先搜尋消息，再點擊「檢視消費軌跡」", viewTrace: "檢視消費軌跡", detailTitle: "消費軌跡詳情", + typePub: "Pub", + typeSubBefore: "SubBefore", + typeSubAfter: "SubAfter", + typeEndTransaction: "EndTransaction", + typeUnknown: "未知", + statusSuccess: "成功", + statusFail: "失敗", + fieldRegionId: "Region", + fieldGroup: "Group", + fieldTopic: "Topic", + fieldMsgId: "MsgId", + fieldTags: "Tags", + fieldKeys: "Keys", + fieldStoreHost: "Store Host", + fieldClientHost: "Client Host", + fieldBodyLength: "Body Length", + fieldCostTime: "耗時", + fieldMsgType: "訊息類型", + fieldOffsetMsgId: "Offset MsgId", + fieldRequestId: "Request ID", + fieldRetryTimes: "重試次數", + fieldContextCode: "Context Code", + rawPayload: "原始軌跡", traceTopicRouteMissing: "軌跡 Topic 在 NameServer 上沒有路由資訊（CODE: 17）。\n\n常見原因：\n1. Broker 未開啟消息軌跡：在 broker.conf 中設置 traceTopicEnable=true 並重新啟動 Broker\n2. Producer/Consumer 未開啟軌跡：setEnableTrace(true)\n3. 使用了自定義軌跡 Topic，請在上方選擇正確的軌跡 Topic", }, }); diff --git a/apps/desktop/src/lib/backend/api.ts b/apps/desktop/src/lib/backend/api.ts index e00df892a..c9a4bcbd4 100644 --- a/apps/desktop/src/lib/backend/api.ts +++ b/apps/desktop/src/lib/backend/api.ts @@ -489,6 +489,7 @@ export const mqListClientConnections = forward("mqListClientConnections"); export const mqListClientChannels = forward("mqListClientChannels"); export const mqCloseClientConnection = forward("mqCloseClientConnection"); export const mqListSubscriptions = forward("mqListSubscriptions"); +export const mqEnrichSubscriptions = forward("mqEnrichSubscriptions"); export const mqCreateSubscription = forward("mqCreateSubscription"); export const mqDeleteSubscription = forward("mqDeleteSubscription"); export const mqSkipMessages = forward("mqSkipMessages"); diff --git a/apps/desktop/src/lib/backend/mq-http.ts b/apps/desktop/src/lib/backend/mq-http.ts index dcab13a92..785906219 100644 --- a/apps/desktop/src/lib/backend/mq-http.ts +++ b/apps/desktop/src/lib/backend/mq-http.ts @@ -217,6 +217,10 @@ export async function mqListSubscriptions(connectionId: string, topic: TopicRef) return post("/api/mq/subscriptions/list", { connectionId, topic }); } +export async function mqEnrichSubscriptions(connectionId: string, topic: TopicRef): Promise { + return post("/api/mq/subscriptions/enrich", { connectionId, topic }); +} + export async function mqCreateSubscription(connectionId: string, topic: TopicRef, sub: string, pos: ResetPosition): Promise { return post("/api/mq/subscriptions/create", { connectionId, topic, sub, pos }); } diff --git a/apps/desktop/src/lib/backend/mq-tauri.ts b/apps/desktop/src/lib/backend/mq-tauri.ts index 8c2dbe5c4..7adf51524 100644 --- a/apps/desktop/src/lib/backend/mq-tauri.ts +++ b/apps/desktop/src/lib/backend/mq-tauri.ts @@ -211,6 +211,10 @@ export async function mqListSubscriptions(connectionId: string, topic: TopicRef) return invoke("mq_list_subscriptions", { connectionId, topic }); } +export async function mqEnrichSubscriptions(connectionId: string, topic: TopicRef): Promise { + return invoke("mq_enrich_subscriptions", { connectionId, topic }); +} + export async function mqCreateSubscription(connectionId: string, topic: TopicRef, sub: string, pos: ResetPosition): Promise { return invoke("mq_create_subscription", { connectionId, topic, sub, pos }); } diff --git a/apps/desktop/src/lib/mq/__tests__/rocketmqConsumerGroupTypes.spec.ts b/apps/desktop/src/lib/mq/__tests__/rocketmqConsumerGroupTypes.spec.ts index 660d00f29..94ab334ff 100644 --- a/apps/desktop/src/lib/mq/__tests__/rocketmqConsumerGroupTypes.spec.ts +++ b/apps/desktop/src/lib/mq/__tests__/rocketmqConsumerGroupTypes.spec.ts @@ -5,13 +5,15 @@ describe("rocketmqConsumerGroupTypes", () => { it("resolves group type from consumerGroupType or subType", () => { expect(resolveRocketMqConsumerGroupType({ consumerGroupType: "FIFO", subType: "NORMAL" })).toBe("FIFO"); expect(resolveRocketMqConsumerGroupType({ subType: "SYSTEM" })).toBe("SYSTEM"); + expect(resolveRocketMqConsumerGroupType({ consumerGroupType: "UNKNOWN" })).toBe("UNKNOWN"); expect(resolveRocketMqConsumerGroupType({ subType: "CONSUME_PASSIVELY ? CLUSTERING" })).toBe("NORMAL"); }); it("filters by dashboard group types", () => { - const filters = { NORMAL: true, FIFO: true, SYSTEM: false }; + const filters = { NORMAL: true, FIFO: true, SYSTEM: false, UNKNOWN: true }; expect(matchesRocketMqConsumerGroupTypeFilters({ consumerGroupType: "NORMAL", subType: "NORMAL" }, filters)).toBe(true); expect(matchesRocketMqConsumerGroupTypeFilters({ consumerGroupType: "SYSTEM", subType: "SYSTEM" }, filters)).toBe(false); + expect(matchesRocketMqConsumerGroupTypeFilters({ consumerGroupType: "UNKNOWN", subType: "UNKNOWN" }, filters)).toBe(true); }); it("resolves message model", () => { diff --git a/apps/desktop/src/lib/mq/__tests__/rocketmqTraceUtils.spec.ts b/apps/desktop/src/lib/mq/__tests__/rocketmqTraceUtils.spec.ts index b03b5cb88..216f661e6 100644 --- a/apps/desktop/src/lib/mq/__tests__/rocketmqTraceUtils.spec.ts +++ b/apps/desktop/src/lib/mq/__tests__/rocketmqTraceUtils.spec.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from "vitest"; -import { formatRocketMqTraceError, isRocketMqTraceTopicRouteMissingError } from "@/lib/mq/rocketmqTraceUtils"; +import { formatRocketMqTraceError, isParsedRocketMqTraceRecord, isRocketMqTraceTopicRouteMissingError, parseRocketMqTracePayload, splitRocketMqTraceKeys } from "@/lib/mq/rocketmqTraceUtils"; + +const SOH = "\u0001"; +const STX = "\u0002"; + +function joinFields(...parts: string[]): string { + return parts.join(SOH); +} describe("rocketmqTraceUtils", () => { it("detects RocketMQ trace topic route missing errors", () => { @@ -11,4 +18,75 @@ describe("rocketmqTraceUtils", () => { const hint = "Enable traceTopicEnable on broker"; expect(formatRocketMqTraceError(new Error("CODE: 17 DESC: No topic route info in name server for the topic: RMQ_SYS_TRACE_TOPIC"), hint)).toBe(hint); }); + + it("parses Pub records including optional clientHost", () => { + const payload = joinFields( + "Pub", + "1785728122381", + "DefaultRegion", + "api-orch-server_producer", + "SIGNATURE_OPERATE_LOG_TOPIC_DEV", + "AC12021900073BCED0D20CC95A0D0197", + "ADD_SIGNATURE_OPERATE_LOG_TAG", + "AC12021900073BCED0D20CC95A0D0197 AC12021900073BCED0D20CC95BFF019F", + "192.168.3.25:10911", + "512", + "4", + "0", + "AC12021900073BCED0D20CC95A0D0197", + "true", + "192.168.2.25", + ); + + const [record] = parseRocketMqTracePayload(payload); + expect(record.type).toBe("Pub"); + expect(record.timestamp).toBe(1785728122381); + expect(record.success).toBe(true); + expect(isParsedRocketMqTraceRecord(record)).toBe(true); + expect(record.fields.find((field) => field.key === "group")?.value).toBe("api-orch-server_producer"); + expect(record.fields.find((field) => field.key === "topic")?.value).toBe("SIGNATURE_OPERATE_LOG_TOPIC_DEV"); + expect(record.fields.find((field) => field.key === "costTime")?.value).toBe("4 ms"); + expect(record.fields.find((field) => field.key === "clientHost")?.value).toBe("192.168.2.25"); + }); + + it("parses SubBefore and SubAfter with version-tolerant lengths", () => { + const subBefore = joinFields("SubBefore", "1785728123000", "DefaultRegion", "LISTING_AGREEMENT_SYNC_TOPIC_LISTING_FULL_AGREEMENT_CONSUMER_DEV", "req-1", "AC12021900073BCED0D20CC95BFF019F", "0", "null", "192.168.2.30"); + const subAfterOld = joinFields("SubAfter", "req-1", "AC12021900073BCED0D20CC95BFF019F", "12", "true", "k1", "0"); + const subAfterNew = joinFields("SubAfter", "req-2", "AC12021900073BCED0D20CC95BFF019F", "20", "false", "k2", "1", "1785728123120", "LISTING_AGREEMENT_SYNC_TOPIC_LISTING_FULL_AGREEMENT_CONSUMER_DEV"); + + const records = parseRocketMqTracePayload(`${subBefore}${STX}${subAfterOld}${STX}${subAfterNew}`); + expect(records).toHaveLength(3); + expect(records[0].type).toBe("SubBefore"); + expect(records[0].fields.find((field) => field.key === "retryTimes")?.value).toBe("0"); + expect(records[0].fields.find((field) => field.key === "keys")?.value).toBe("-"); + expect(records[0].fields.find((field) => field.key === "clientHost")?.value).toBe("192.168.2.30"); + + expect(records[1].type).toBe("SubAfter"); + expect(records[1].success).toBe(true); + expect(records[1].timestamp).toBeUndefined(); + expect(records[1].fields.find((field) => field.key === "costTime")?.value).toBe("12 ms"); + + expect(records[2].type).toBe("SubAfter"); + expect(records[2].success).toBe(false); + expect(records[2].timestamp).toBe(1785728123120); + expect(records[2].fields.find((field) => field.key === "group")?.value).toBe("LISTING_AGREEMENT_SYNC_TOPIC_LISTING_FULL_AGREEMENT_CONSUMER_DEV"); + }); + + it("normalizes Type@ prefix and falls back for unknown payloads", () => { + const withAt = `Pub@1785728122381${SOH}DefaultRegion${SOH}g${SOH}t${SOH}m${SOH}tag${SOH}k${SOH}host${SOH}1${SOH}2${SOH}0${SOH}off${SOH}true`; + const [parsed] = parseRocketMqTracePayload(withAt); + expect(parsed.type).toBe("Pub"); + expect(parsed.fields.find((field) => field.key === "group")?.value).toBe("g"); + + const [unknown] = parseRocketMqTracePayload("not-a-trace-payload"); + expect(unknown.type).toBe("Unknown"); + expect(isParsedRocketMqTraceRecord(unknown)).toBe(false); + expect(unknown.raw).toBe("not-a-trace-payload"); + }); + + it("splits KEYS into chips", () => { + expect(splitRocketMqTraceKeys("a b,c")).toEqual(["a", "b", "c"]); + expect(splitRocketMqTraceKeys("null")).toEqual([]); + expect(splitRocketMqTraceKeys("-")).toEqual([]); + }); }); diff --git a/apps/desktop/src/lib/mq/rocketmqConsumerGroupTypes.ts b/apps/desktop/src/lib/mq/rocketmqConsumerGroupTypes.ts index 578e3f3ae..b6f8a3e22 100644 --- a/apps/desktop/src/lib/mq/rocketmqConsumerGroupTypes.ts +++ b/apps/desktop/src/lib/mq/rocketmqConsumerGroupTypes.ts @@ -1,19 +1,21 @@ import type { SubscriptionInfo } from "@/types/mq"; -export type RocketMqConsumerGroupType = "NORMAL" | "FIFO" | "SYSTEM"; +export type RocketMqConsumerGroupType = "NORMAL" | "FIFO" | "SYSTEM" | "UNKNOWN"; -export const ROCKETMQ_CONSUMER_GROUP_TYPES: readonly RocketMqConsumerGroupType[] = ["NORMAL", "FIFO", "SYSTEM"]; +export const ROCKETMQ_CONSUMER_GROUP_TYPES: readonly RocketMqConsumerGroupType[] = ["NORMAL", "FIFO", "SYSTEM", "UNKNOWN"]; export const DEFAULT_ROCKETMQ_CONSUMER_GROUP_TYPE_FILTERS: Record = { NORMAL: true, FIFO: true, SYSTEM: false, + UNKNOWN: true, }; export function resolveRocketMqConsumerGroupType(sub: Pick): RocketMqConsumerGroupType { const raw = (sub.consumerGroupType ?? sub.subType)?.trim().toUpperCase(); if (raw === "FIFO" || raw === "ORDER" || raw === "ORDERLY") return "FIFO"; if (raw === "SYSTEM") return "SYSTEM"; + if (raw === "UNKNOWN") return "UNKNOWN"; if (raw === "NORMAL") return "NORMAL"; return "NORMAL"; } diff --git a/apps/desktop/src/lib/mq/rocketmqTraceUtils.ts b/apps/desktop/src/lib/mq/rocketmqTraceUtils.ts index 9aacec0f0..e2177ef18 100644 --- a/apps/desktop/src/lib/mq/rocketmqTraceUtils.ts +++ b/apps/desktop/src/lib/mq/rocketmqTraceUtils.ts @@ -1,4 +1,5 @@ import { formatError } from "@/lib/backend/errorUtils"; +import { formatRocketMqTimestamp } from "@/lib/mq/rocketmqMessageUtils"; /** RocketMQ CODE 17: trace topic has no route on NameServer (trace not enabled or wrong topic). */ export function isRocketMqTraceTopicRouteMissingError(error: unknown): boolean { @@ -12,3 +13,184 @@ export function formatRocketMqTraceError(error: unknown, traceTopicMissingHint: } return formatError(error); } + +/** Content / field splitters from Apache RocketMQ TraceConstants. */ +const CONTENT_SPLITTER = "\u0001"; +const FIELD_SPLITTER = "\u0002"; + +export type RocketMqTraceType = "Pub" | "SubBefore" | "SubAfter" | "EndTransaction" | "Unknown"; + +/** Stable field keys mapped to mqTrace.field* i18n labels in the dialog. */ +export type RocketMqTraceFieldKey = "regionId" | "group" | "topic" | "msgId" | "tags" | "keys" | "storeHost" | "clientHost" | "bodyLength" | "costTime" | "msgType" | "offsetMsgId" | "requestId" | "retryTimes" | "contextCode"; + +export interface RocketMqTraceField { + key: RocketMqTraceFieldKey; + value: string; +} + +export interface RocketMqTraceRecord { + type: RocketMqTraceType; + timestamp?: number; + /** Parsed success flag when present (Pub / SubAfter). */ + success?: boolean; + fields: RocketMqTraceField[]; + raw: string; +} + +function emptyDisplay(value: string | undefined): string { + if (value == null) return "-"; + const trimmed = value.trim(); + if (!trimmed || trimmed === "null") return "-"; + return trimmed; +} + +function parseTimestamp(raw: string | undefined): number | undefined { + if (!raw?.trim()) return undefined; + const numeric = Number(raw); + return Number.isFinite(numeric) ? numeric : undefined; +} + +function parseSuccess(raw: string | undefined): boolean | undefined { + if (raw == null || !raw.trim()) return undefined; + const normalized = raw.trim().toLowerCase(); + if (normalized === "true") return true; + if (normalized === "false") return false; + return undefined; +} + +function formatCostTime(raw: string | undefined): string { + const value = emptyDisplay(raw); + if (value === "-") return value; + const numeric = Number(value); + if (!Number.isFinite(numeric)) return value; + return `${numeric} ms`; +} + +function pushField(fields: RocketMqTraceField[], key: RocketMqTraceFieldKey, value: string | undefined): void { + const display = key === "costTime" ? formatCostTime(value) : emptyDisplay(value); + // Keep tags/keys/retry visible even when empty so Pub/Sub cards stay scannable. + if (display === "-" && key !== "tags" && key !== "keys" && key !== "retryTimes") return; + fields.push({ key, value: display }); +} + +/** Normalize rare "Type@fields" prefix into official Type\\u0001fields form. */ +function normalizeTraceSegment(segment: string): string { + return segment.replace(/^(Pub|SubBefore|SubAfter|EndTransaction)@/, `$1${CONTENT_SPLITTER}`); +} + +function parsePub(parts: string[], raw: string): RocketMqTraceRecord { + // Pub: type, ts, region, group, topic, msgId, tags, keys, storeHost, bodyLength, cost, msgType, offsetMsgId, success[, clientHost] + const timestamp = parseTimestamp(parts[1]); + const success = parseSuccess(parts[13]); + const fields: RocketMqTraceField[] = []; + pushField(fields, "regionId", parts[2]); + pushField(fields, "group", parts[3]); + pushField(fields, "topic", parts[4]); + pushField(fields, "msgId", parts[5]); + pushField(fields, "tags", parts[6]); + pushField(fields, "keys", parts[7]); + pushField(fields, "storeHost", parts[8]); + pushField(fields, "bodyLength", parts[9]); + pushField(fields, "costTime", parts[10]); + pushField(fields, "msgType", parts[11]); + pushField(fields, "offsetMsgId", parts[12]); + if (parts.length >= 15) pushField(fields, "clientHost", parts[14]); + return { type: "Pub", timestamp, success, fields, raw }; +} + +function parseSubBefore(parts: string[], raw: string): RocketMqTraceRecord { + // SubBefore: type, ts, region, group, requestId, msgId, retryTimes, keys[, clientHost] + // Some encoders also append storeHost before clientHost (length >= 9/10). + const timestamp = parseTimestamp(parts[1]); + const fields: RocketMqTraceField[] = []; + pushField(fields, "regionId", parts[2]); + pushField(fields, "group", parts[3]); + pushField(fields, "requestId", parts[4]); + pushField(fields, "msgId", parts[5]); + pushField(fields, "retryTimes", parts[6]); + pushField(fields, "keys", parts[7]); + if (parts.length >= 10) { + pushField(fields, "storeHost", parts[8]); + pushField(fields, "clientHost", parts[9]); + } else if (parts.length >= 9) { + pushField(fields, "clientHost", parts[8]); + } + return { type: "SubBefore", timestamp, fields, raw }; +} + +function parseSubAfter(parts: string[], raw: string): RocketMqTraceRecord { + // SubAfter: type, requestId, msgId, cost, success, keys, contextCode[, timestamp, groupName] + const success = parseSuccess(parts[4]); + const timestamp = parts.length >= 8 ? parseTimestamp(parts[7]) : undefined; + const fields: RocketMqTraceField[] = []; + pushField(fields, "requestId", parts[1]); + pushField(fields, "msgId", parts[2]); + pushField(fields, "costTime", parts[3]); + pushField(fields, "keys", parts[5]); + pushField(fields, "contextCode", parts[6]); + if (parts.length >= 9) pushField(fields, "group", parts[8]); + return { type: "SubAfter", timestamp, success, fields, raw }; +} + +function parseEndTransaction(parts: string[], raw: string): RocketMqTraceRecord { + // EndTransaction: type, ts, region, group, topic, msgId, tags, keys, storeHost, msgType, clientHost, ... + const timestamp = parseTimestamp(parts[1]); + const fields: RocketMqTraceField[] = []; + pushField(fields, "regionId", parts[2]); + pushField(fields, "group", parts[3]); + pushField(fields, "topic", parts[4]); + pushField(fields, "msgId", parts[5]); + pushField(fields, "tags", parts[6]); + pushField(fields, "keys", parts[7]); + pushField(fields, "storeHost", parts[8]); + pushField(fields, "msgType", parts[9]); + if (parts.length >= 11) pushField(fields, "clientHost", parts[10]); + return { type: "EndTransaction", timestamp, fields, raw }; +} + +function parseTraceSegment(segment: string): RocketMqTraceRecord { + const raw = segment; + const normalized = normalizeTraceSegment(segment.trim()); + if (!normalized) { + return { type: "Unknown", fields: [], raw }; + } + const parts = normalized.split(CONTENT_SPLITTER); + const type = parts[0]?.trim(); + if (type === "Pub" && parts.length >= 14) return parsePub(parts, raw); + if (type === "SubBefore" && parts.length >= 8) return parseSubBefore(parts, raw); + if (type === "SubAfter" && parts.length >= 7) return parseSubAfter(parts, raw); + if (type === "EndTransaction" && parts.length >= 9) return parseEndTransaction(parts, raw); + return { type: "Unknown", fields: [], raw }; +} + +/** + * Parse RocketMQ trace message body into structured records. + * Official format: Type\\u0001field...\\u0002Type\\u0001... + */ +export function parseRocketMqTracePayload(payload: string): RocketMqTraceRecord[] { + if (!payload?.trim()) return []; + const segments = payload + .split(FIELD_SPLITTER) + .map((part) => part.trim()) + .filter(Boolean); + // Body may be a single record without trailing STX. + const units = segments.length ? segments : [payload.trim()]; + return units.map(parseTraceSegment); +} + +export function isParsedRocketMqTraceRecord(record: RocketMqTraceRecord): boolean { + return record.type !== "Unknown" && record.fields.length > 0; +} + +/** Split KEYS header / keys field into display chips. */ +export function splitRocketMqTraceKeys(value: string | undefined): string[] { + if (!value?.trim() || value.trim() === "-" || value.trim() === "null") return []; + return value + .split(/[\s,]+/) + .map((part) => part.trim()) + .filter(Boolean); +} + +export function rocketMqTraceRecordTimeLabel(record: RocketMqTraceRecord, fallbackTimestamp?: number): string { + return formatRocketMqTimestamp(record.timestamp ?? fallbackTimestamp); +} diff --git a/apps/desktop/src/types/mq.ts b/apps/desktop/src/types/mq.ts index fed402195..3c16fc11c 100644 --- a/apps/desktop/src/types/mq.ts +++ b/apps/desktop/src/types/mq.ts @@ -192,6 +192,8 @@ export interface SubscriptionInfo { consumerGroupType?: string; /** RocketMQ consumer group message model: CLUSTERING / BROADCASTING. */ messageModel?: string; + /** When true, backlog probe failed — do not render msgBacklog as healthy zero. */ + backlogUnavailable?: boolean; } export interface RocketMqConsumerGroupConfig { @@ -228,9 +230,25 @@ export type ResetPosition = { kind: "earliest" } | { kind: "latest" } | { kind: export type SkipCount = { kind: "all" } | { kind: "count"; count: number }; +/** Per-queue consume progress (RocketMQ Dashboard consume-detail / Kafka lag rows). */ +export interface PartitionBacklog { + partition: number; + /** Consumer committed offset (`consumerOffset`). */ + currentOffset: number; + /** Broker max offset (`brokerOffset`). */ + endOffset: number; + lag: number; + brokerName?: string; + /** Last consume message store timestamp (ms). `0` means unavailable. */ + lastTimestamp?: number; + consumerClient?: string; +} + export interface BacklogStats { msgBacklog: number; backlogSize: number; + /** Optional queue-level progress; omitted or empty when adapter only exposes totals. */ + partitions?: PartitionBacklog[]; } export interface ClusterInfo { diff --git a/crates/dbx-core/src/connection.rs b/crates/dbx-core/src/connection.rs index ec9cce519..ec7e9178c 100644 --- a/crates/dbx-core/src/connection.rs +++ b/crates/dbx-core/src/connection.rs @@ -1292,6 +1292,12 @@ impl AppState { if wait_for_drain { self.wait_for_pool_drain(&pool_key).await; } + #[cfg(feature = "mq-admin")] + let mq_keepalive_adapter = if matches!(&pool, PoolKind::MessageQueue) { + self.mq_registry.get_cached_adapter(&config.id).await + } else { + None + }; let routing = self.pool_routing_control(); let previous = loop { let mut connections = self.connections.write().await; @@ -1309,7 +1315,13 @@ impl AppState { // candidate never reaches this point, so the existing route keeps its state. routing.stop_keepalive(&pool_key); activity.insert(pool_key.clone(), PoolActivity::now()); - self.start_keepalive_task(&pool_key, &pool, config); + self.start_keepalive_task( + &pool_key, + &pool, + config, + #[cfg(feature = "mq-admin")] + mq_keepalive_adapter.clone(), + ); break Ok(connections.insert(pool_key.clone(), pool)); }; let previous = match previous { @@ -1430,9 +1442,21 @@ impl AppState { self.pool_routing_control().close_pool_with_timeout(pool_key, pool).await; } - fn start_keepalive_task(&self, pool_key: &str, pool: &PoolKind, config: &ConnectionConfig) { + fn start_keepalive_task( + &self, + pool_key: &str, + pool: &PoolKind, + config: &ConnectionConfig, + #[cfg(feature = "mq-admin")] mq_adapter: Option>, + ) { let interval_secs = config.keepalive_interval_secs; let mut target = keepalive_target_from_pool(pool, config); + #[cfg(feature = "mq-admin")] + if target.is_none() { + if let Some(adapter) = mq_adapter { + target = Some(KeepaliveTarget::MessageQueue(adapter)); + } + } if interval_secs == 0 { return; } @@ -1445,10 +1469,28 @@ impl AppState { let key = pool_key.to_string(); let interval = Duration::from_secs(interval_secs.max(1)); + // MQ keepalive runs a full adapter test_connection (agent RPC); use query timeout + // so slow clusters are not spuriously dropped by the shorter connect timeout. + #[cfg(feature = "mq-admin")] + let timeout = if matches!(target, Some(KeepaliveTarget::MessageQueue(_))) { + match config.effective_query_timeout_secs() { + 0 => Duration::from_secs(300), // UI "unlimited" still needs a keepalive bound + secs => Duration::from_secs(secs.max(1)), + } + } else { + Duration::from_secs(config.effective_connect_timeout_secs().max(1)) + }; + #[cfg(not(feature = "mq-admin"))] let timeout = Duration::from_secs(config.effective_connect_timeout_secs().max(1)); let routing = self.pool_routing_control(); let connections = self.connections.clone(); let running_queries = self.running_queries.clone(); + // MQ pool markers are empty; close_pool_kind is a no-op, so keepalive must + // drop the registry adapter or reconnect would reuse a dead agent. + #[cfg(feature = "mq-admin")] + let mq_registry = self.mq_registry.clone(); + #[cfg(feature = "mq-admin")] + let mq_connection_id = config.id.clone(); self.task_supervisor.spawn_replace(format!("keepalive:{pool_key}"), move |shutdown| async move { loop { tokio::select! { @@ -1468,6 +1510,17 @@ impl AppState { log::warn!("Connection keepalive failed for '{key}': {err}; invalidating pool"); let replace_runtime = err.recovery_decision().is_some_and(RecoveryDecision::replaces_runtime); + // PoolKind::MessageQueue is a unit marker, so matches_pool cannot + // Arc::ptr_eq. Identity-check the registry adapter before teardown. + #[cfg(feature = "mq-admin")] + if let KeepaliveTarget::MessageQueue(adapter) = target { + if !mq_registry.is_current_adapter(&mq_connection_id, adapter).await { + log::debug!("Skipping stale MQ keepalive result for replaced pool '{key}'"); + break; + } + } + #[cfg(feature = "mq-admin")] + let drop_mq = matches!(target, KeepaliveTarget::MessageQueue(_)); if !detach_keepalive_target_if_current( &routing, &connections, @@ -1478,6 +1531,11 @@ impl AppState { .await { log::debug!("Skipping stale keepalive result for replaced pool '{key}'"); + } else { + #[cfg(feature = "mq-admin")] + if drop_mq { + mq_registry.drop_connection(&mq_connection_id).await; + } } break; } @@ -1486,8 +1544,22 @@ impl AppState { "Connection keepalive timed out for '{key}' after {}s; invalidating pool", timeout.as_secs() ); + #[cfg(feature = "mq-admin")] + if let KeepaliveTarget::MessageQueue(adapter) = target { + if !mq_registry.is_current_adapter(&mq_connection_id, adapter).await { + log::debug!("Skipping stale MQ keepalive timeout for replaced pool '{key}'"); + break; + } + } + #[cfg(feature = "mq-admin")] + let drop_mq = matches!(target, KeepaliveTarget::MessageQueue(_)); if !detach_keepalive_target_if_current(&routing, &connections, &key, target, false).await { log::debug!("Skipping stale keepalive timeout for replaced pool '{key}'"); + } else { + #[cfg(feature = "mq-admin")] + if drop_mq { + mq_registry.drop_connection(&mq_connection_id).await; + } } break; } @@ -3369,7 +3441,13 @@ impl AppState { } }; if let (Some(config), Some(client)) = (config, client) { - self.start_keepalive_task(pool_key, &PoolKind::Agent(client), &config); + self.start_keepalive_task( + pool_key, + &PoolKind::Agent(client), + &config, + #[cfg(feature = "mq-admin")] + None, + ); } } @@ -4083,7 +4161,10 @@ enum KeepaliveTarget { Postgres(deadpool_postgres::Pool), Rqlite(db::rqlite_driver::RqliteClient), Turso(db::turso_driver::TursoClient), - MongoDb { client: mongodb::Client, database: Option }, + MongoDb { + client: mongodb::Client, + database: Option, + }, ClickHouse(db::clickhouse_driver::ChClient), SqlServer(Arc>), Elasticsearch(db::elasticsearch_driver::EsClient), @@ -4093,6 +4174,8 @@ enum KeepaliveTarget { InfluxDb(db::influxdb_driver::InfluxdbClient), VictoriaMetrics(db::victoriametrics_driver::VictoriaMetricsClient), Agent(Arc), + #[cfg(feature = "mq-admin")] + MessageQueue(Arc), } #[derive(Debug)] @@ -4130,6 +4213,10 @@ impl KeepaliveTarget { match (self, pool) { (Self::Agent(expected), PoolKind::Agent(current)) => Arc::ptr_eq(expected, current), (Self::SqlServer(expected), PoolKind::SqlServer(current)) => Arc::ptr_eq(expected, current), + #[cfg(feature = "mq-admin")] + (Self::MessageQueue(_), PoolKind::MessageQueue) => true, + #[cfg(feature = "mq-admin")] + (Self::MessageQueue(_), _) | (_, PoolKind::MessageQueue) => false, (Self::Agent(_), _) | (_, PoolKind::Agent(_)) | (Self::SqlServer(_), _) | (_, PoolKind::SqlServer(_)) => { false } @@ -4245,6 +4332,10 @@ async fn ping_keepalive_target(target: &mut KeepaliveTarget, timeout: Duration) Err(error) => Err(KeepaliveError::Agent(error)), } } + #[cfg(feature = "mq-admin")] + KeepaliveTarget::MessageQueue(adapter) => { + adapter.test_connection().await.map(|_| ()).map_err(KeepaliveError::from) + } } } @@ -6553,7 +6644,13 @@ for line in sys.stdin: .await .insert(pool_key.to_string(), super::PoolActivity::idle_for(std::time::Duration::from_secs(10))); let pool = super::clone_pool_kind(state.connections.read().await.get(pool_key).unwrap()); - state.start_keepalive_task(pool_key, &pool, &config); + state.start_keepalive_task( + pool_key, + &pool, + &config, + #[cfg(feature = "mq-admin")] + None, + ); tokio::time::sleep(std::time::Duration::from_millis(20)).await; diff --git a/crates/dbx-core/src/mq/adapters/kafka.rs b/crates/dbx-core/src/mq/adapters/kafka.rs index 4f9e3cb19..98c4caf77 100644 --- a/crates/dbx-core/src/mq/adapters/kafka.rs +++ b/crates/dbx-core/src/mq/adapters/kafka.rs @@ -63,12 +63,13 @@ impl KafkaAdmin { let mut client = AgentDriverClient::spawn(launch).await?; // Handshake - let _: serde_json::Value = client.call("handshake", serde_json::json!({})).await?; + let _: serde_json::Value = + client.call_with_timeout("handshake", serde_json::json!({}), cfg.rpc_timeout()).await?; // Build the connection params from MqAdminConfig let conn_params = build_connection_params(&cfg); let connect_params = serde_json::json!({ "connection": conn_params }); - let _: serde_json::Value = client.call("connect", connect_params).await?; + let _: serde_json::Value = client.call_with_timeout("connect", connect_params, cfg.rpc_timeout()).await?; log::info!("Kafka admin connected via agent (bootstrap servers: {})", bootstrap_servers(&cfg)); @@ -82,7 +83,7 @@ impl KafkaAdmin { params: serde_json::Value, ) -> Result { let mut client = self.client.lock().await; - client.call(method, params).await + client.call_with_timeout(method, params, self.config.rpc_timeout()).await } /// The Kafka agent bounds message browsing with its configured request timeout. @@ -538,8 +539,7 @@ impl MessageQueueAdmin for KafkaAdmin { ) .await?; - let total_lag = result.get("totalLag").and_then(|v| v.as_i64()).unwrap_or(0); - Ok(BacklogStats { msg_backlog: total_lag, backlog_size: 0 }) + Ok(backlog_stats_from_consumer_lag(&result)) } async fn get_cluster_info(&self) -> Result { @@ -655,6 +655,7 @@ fn build_connection_params(cfg: &MqAdminConfig) -> serde_json::Value { "skip_verify": cfg.tls_skip_verify, }, "properties": properties, + "request_timeout_ms": cfg.request_timeout_ms(), }) } @@ -802,6 +803,8 @@ mod tests { token_signing: None, connect_override: None, management_connect_override: None, + query_timeout_secs: crate::mq::config::DEFAULT_MQ_QUERY_TIMEOUT_SECS, + connect_timeout_secs: crate::mq::config::DEFAULT_MQ_CONNECT_TIMEOUT_SECS, extra, } } diff --git a/crates/dbx-core/src/mq/adapters/pulsar.rs b/crates/dbx-core/src/mq/adapters/pulsar.rs index ee5385afb..dc4b18d75 100644 --- a/crates/dbx-core/src/mq/adapters/pulsar.rs +++ b/crates/dbx-core/src/mq/adapters/pulsar.rs @@ -16,8 +16,6 @@ use crate::mq::port::MessageQueueAdmin; use crate::mq::types::*; use crate::mq::util::truncate; -/// How long to wait for a single admin REST call. -const REQUEST_TIMEOUT_SECS: u64 = 30; /// How long to wait for the initial version-probe during construction. const PROBE_TIMEOUT_SECS: u64 = 8; const DETAIL_REQUEST_CONCURRENCY: usize = 8; @@ -39,7 +37,12 @@ impl PulsarAdmin { /// the API profile. Probe failure degrades gracefully to the 3.1.x baseline. pub async fn new(cfg: MqAdminConfig) -> Result { let base = normalize_base(&cfg.admin_url); - let http = build_http_client(cfg.tls_skip_verify, cfg.connect_override.as_ref(), &cfg.admin_url)?; + // Unlimited query timeout still needs a finite HTTP client timeout; mirror + // request_timeout_ms (1h) so Advanced "0 = unlimited" is not silently clamped to 30s. + let request_timeout = + cfg.rpc_timeout().unwrap_or_else(|| std::time::Duration::from_millis(cfg.request_timeout_ms())); + let http = + build_http_client(cfg.tls_skip_verify, cfg.connect_override.as_ref(), &cfg.admin_url, request_timeout)?; let auth = cfg.auth.clone(); let token_cache = TokenCache::default(); @@ -807,9 +810,9 @@ fn build_http_client( tls_skip_verify: bool, connect_override: Option<&MqConnectOverride>, admin_url: &str, + request_timeout: std::time::Duration, ) -> Result { - let mut builder = crate::db::http_client_builder(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS)) - .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS)); + let mut builder = crate::db::http_client_builder(request_timeout).timeout(request_timeout); if tls_skip_verify { builder = builder.danger_accept_invalid_certs(true); } @@ -1074,7 +1077,7 @@ fn backlog_stats_from_topic_stats(profile: &PulsarApiProfile, stats: &TopicStats None => subs.into_iter().fold(0_i64, |total, sub| total.saturating_add(sub.msg_backlog)), }; - BacklogStats { msg_backlog, backlog_size: stats.backlog_size } + BacklogStats { msg_backlog, backlog_size: stats.backlog_size, partitions: Vec::new() } } #[cfg(test)] @@ -1151,6 +1154,8 @@ mod tests { token_signing: None, connect_override: None, management_connect_override: None, + query_timeout_secs: crate::mq::config::DEFAULT_MQ_QUERY_TIMEOUT_SECS, + connect_timeout_secs: crate::mq::config::DEFAULT_MQ_CONNECT_TIMEOUT_SECS, extra: serde_json::Value::Null, }) .await diff --git a/crates/dbx-core/src/mq/adapters/rabbitmq.rs b/crates/dbx-core/src/mq/adapters/rabbitmq.rs index 5ae13883c..b4f995629 100644 --- a/crates/dbx-core/src/mq/adapters/rabbitmq.rs +++ b/crates/dbx-core/src/mq/adapters/rabbitmq.rs @@ -64,12 +64,13 @@ impl RabbitMqAdmin { let mut client = AgentDriverClient::spawn(launch).await?; // Handshake - let _: serde_json::Value = client.call("handshake", serde_json::json!({})).await?; + let _: serde_json::Value = + client.call_with_timeout("handshake", serde_json::json!({}), cfg.rpc_timeout()).await?; // Build the connection params from MqAdminConfig let conn_params = build_connection_params(&cfg)?; let connect_params = serde_json::json!({ "connection": conn_params }); - let _: serde_json::Value = client.call("connect", connect_params).await?; + let _: serde_json::Value = client.call_with_timeout("connect", connect_params, cfg.rpc_timeout()).await?; log::info!("RabbitMQ admin connected via agent (addresses: {})", addresses(&cfg)); @@ -83,7 +84,7 @@ impl RabbitMqAdmin { params: serde_json::Value, ) -> Result { let mut client = self.client.lock().await; - client.call(method, params).await + client.call_with_timeout(method, params, self.config.rpc_timeout()).await } /// Send a JSON-RPC call that returns `{ok: true}` on success. @@ -623,7 +624,7 @@ impl MessageQueueAdmin for RabbitMqAdmin { let result: serde_json::Value = self.call("mq_get_topic_stats", params).await?; let total_messages = result.get("totalMessages").and_then(|v| v.as_i64()).unwrap_or(0); - Ok(BacklogStats { msg_backlog: total_messages, backlog_size: total_messages }) + Ok(BacklogStats { msg_backlog: total_messages, backlog_size: total_messages, partitions: Vec::new() }) } async fn get_cluster_info(&self) -> Result { @@ -952,6 +953,7 @@ fn build_connection_params(cfg: &MqAdminConfig) -> Result MqClusterInfo { impl RocketMqAdmin { /// Spawn the RocketMQ Java agent, perform handshake, and connect. + /// + /// Callers must probe NameServer reachability before invoking this so the + /// connect-timeout wall covers only JVM spawn + handshake + connect. pub async fn new(cfg: MqAdminConfig, launch: AgentLaunchSpec) -> Result { let mut client = AgentDriverClient::spawn(launch).await?; // Handshake - let _: serde_json::Value = client.call("handshake", serde_json::json!({})).await?; + let _: serde_json::Value = + client.call_with_timeout("handshake", serde_json::json!({}), cfg.rpc_timeout()).await?; // Build the connection params from MqAdminConfig let conn_params = build_connection_params(&cfg); let connect_params = serde_json::json!({ "connection": conn_params }); - let _: serde_json::Value = client.call("connect", connect_params).await?; + let _: serde_json::Value = client.call_with_timeout("connect", connect_params, cfg.rpc_timeout()).await?; log::info!("RocketMQ admin connected via agent (namesrv: {})", namesrv_addr(&cfg)); @@ -117,7 +125,7 @@ impl RocketMqAdmin { params: serde_json::Value, ) -> Result { let mut client = self.client.lock().await; - client.call(method, params).await + client.call_with_timeout(method, params, self.config.rpc_timeout()).await } /// Send a JSON-RPC call that returns `{ok: true}` on success. @@ -355,13 +363,14 @@ impl MessageQueueAdmin for RocketMqAdmin { async fn list_subscriptions(&self, topic: &TopicRef) -> Result, String> { if topic.topic.is_empty() { + // Fast path: names/types only. UI enrichs online members/topics in a second pass. let result: serde_json::Value = self .call( "mq_list_consumer_groups", serde_json::json!({ "limit": 500, "offset": 0, - "enrich": true, + "enrich": false, }), ) .await?; @@ -369,8 +378,7 @@ impl MessageQueueAdmin for RocketMqAdmin { return Ok(groups.iter().map(rocketmq_subscription_from_group).collect()); } - // Agent uses queryTopicConsumeByWho (Dashboard queryTopicConsumerInfo); include all - // returned groups even when consumers are offline and offsets are zero. + // Topic path: skip list enrich; batch lag in one agent RPC (no N+1). let result: serde_json::Value = self .call( "mq_list_consumer_groups", @@ -378,34 +386,30 @@ impl MessageQueueAdmin for RocketMqAdmin { "topic": topic.topic, "limit": 200, "offset": 0, - "enrich": true, + "enrich": false, + "includeLag": true, }), ) .await?; let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default(); + Ok(groups.iter().map(rocketmq_subscription_from_group).collect()) + } - let mut subs = Vec::new(); - for group in groups { - let group_id = group.get("groupId").and_then(|v| v.as_str()).unwrap_or_default(); - if group_id.is_empty() { - continue; - } - let mut sub = rocketmq_subscription_from_group(&group); - if let Ok(lag) = self - .call::( - "mq_get_consumer_lag", - serde_json::json!({ - "groupId": group_id, - "topic": topic.topic, - }), - ) - .await - { - sub.msg_backlog = lag.get("totalLag").and_then(|v| v.as_i64()).unwrap_or(0); - } - subs.push(sub); + async fn enrich_subscriptions(&self, topic: &TopicRef) -> Result, String> { + // Cluster-wide second pass fills memberCount/topics after the fast list paint. + let mut params = serde_json::json!({ + "limit": 500, + "offset": 0, + "enrich": true, + }); + if !topic.topic.is_empty() { + params["topic"] = serde_json::json!(topic.topic); + params["limit"] = serde_json::json!(200); + params["includeLag"] = serde_json::json!(true); } - Ok(subs) + let result: serde_json::Value = self.call("mq_list_consumer_groups", params).await?; + let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default(); + Ok(groups.iter().map(rocketmq_subscription_from_group).collect()) } async fn create_subscription(&self, _topic: &TopicRef, _sub: &str, _pos: ResetPosition) -> Result<(), String> { @@ -673,8 +677,12 @@ impl MessageQueueAdmin for RocketMqAdmin { ) .await?; - let total_lag = result.get("totalLag").and_then(|v| v.as_i64()).unwrap_or(0); - Ok(BacklogStats { msg_backlog: total_lag, backlog_size: 0 }) + // Agent omits totalLag on probe failure; never coerce that to healthy zero backlog. + if result.get("totalLag").and_then(|v| v.as_i64()).is_none() { + return Err(format!("RocketMQ consumer lag unavailable for group '{group_id}' on topic '{}'", topic.topic)); + } + + Ok(backlog_stats_from_consumer_lag(&result)) } async fn get_cluster_info(&self) -> Result { @@ -831,6 +839,7 @@ fn build_connection_params(cfg: &MqAdminConfig) -> serde_json::Value { "access_key": access_key, "secret_key": secret_key, "tls_skip_verify": cfg.tls_skip_verify, + "request_timeout_ms": cfg.request_timeout_ms(), }) } @@ -901,12 +910,14 @@ fn rocketmq_subscription_for_topic( online_members: None, consumer_group_type: None, message_model: None, + backlog_unavailable: None, }) } fn rocketmq_subscription_from_group(group: &serde_json::Value) -> SubscriptionInfo { let group_id = group.get("groupId").and_then(|v| v.as_str()).unwrap_or_default(); - let group_type = group.get("groupType").and_then(|v| v.as_str()).unwrap_or("NORMAL").to_string(); + // Match agent classify: missing dump → UNKNOWN, not silent NORMAL (FIFO hide risk). + let group_type = group.get("groupType").and_then(|v| v.as_str()).unwrap_or("UNKNOWN").to_string(); let message_model = group.get("messageModel").and_then(|v| v.as_str()).map(String::from); let online_members = group.get("memberCount").and_then(|v| v.as_u64()).map(|v| v as u32); let topics = group @@ -914,10 +925,13 @@ fn rocketmq_subscription_from_group(group: &serde_json::Value) -> SubscriptionIn .and_then(|v| v.as_array()) .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect::>()) .unwrap_or_default(); + let lag_failed = group.get("totalLagFailed").and_then(|v| v.as_bool()).unwrap_or(false); + let total_lag = group.get("totalLag").and_then(|v| v.as_i64()); SubscriptionInfo { name: group_id.to_string(), sub_type: group_type.clone(), - msg_backlog: 0, + // Probe failure: keep 0 but set backlog_unavailable so topic-list UI shows "-". + msg_backlog: total_lag.unwrap_or(0), msg_rate_out: 0.0, msg_throughput_out: 0.0, consumers: Vec::new(), @@ -925,9 +939,92 @@ fn rocketmq_subscription_from_group(group: &serde_json::Value) -> SubscriptionIn online_members, consumer_group_type: Some(group_type), message_model, + backlog_unavailable: if lag_failed || (group.get("totalLag").is_some() && total_lag.is_none()) { + Some(true) + } else { + None + }, } } +/// Fail fast on unreachable NameServer before paying for JVM agent startup. +/// Called outside the connect-timeout wall so the probe does not steal JVM budget. +pub(crate) async fn probe_namesrv_before_connect(cfg: &MqAdminConfig, budget: Duration) -> Result<(), String> { + probe_namesrv_tcp(&namesrv_addr(cfg), budget).await +} + +/// Probe NameServer addresses so connect fails before spawning the JVM agent. +/// Tries each `;`-separated host (and each resolved IP) within the shared budget, +/// matching RocketMQ client HA behavior instead of failing on the first dead node. +async fn probe_namesrv_tcp(namesrv: &str, budget: Duration) -> Result<(), String> { + let hosts: Vec = + namesrv.split(';').map(str::trim).filter(|part| !part.is_empty()).map(str::to_string).collect(); + if hosts.is_empty() { + return Err("RocketMQ namesrv_addr is empty".to_string()); + } + + let deadline = tokio::time::Instant::now() + budget; + // Multi-NS HA: cap each attempt so a blackholed first node leaves budget for later hosts. + let per_attempt = if hosts.len() <= 1 { + budget + } else { + (budget / hosts.len() as u32).clamp(Duration::from_millis(500), Duration::from_secs(3)) + }; + let mut last_error = String::new(); + for host in &hosts { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + if remaining.is_zero() { + break; + } + // DNS resolution is blocking; keep it off the async runtime. + let host_for_resolve = host.clone(); + let addrs = match tokio::task::spawn_blocking(move || { + host_for_resolve + .to_socket_addrs() + .map(|iter| iter.collect::>()) + .map_err(|e| format!("RocketMQ NameServer address '{host_for_resolve}' is invalid: {e}")) + }) + .await + { + Ok(Ok(addrs)) => addrs, + Ok(Err(e)) => { + last_error = e; + continue; + } + Err(e) => { + last_error = format!("RocketMQ NameServer resolve task failed: {e}"); + continue; + } + }; + if addrs.is_empty() { + last_error = format!("RocketMQ NameServer address '{host}' did not resolve"); + continue; + } + for addr in addrs { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + if remaining.is_zero() { + break; + } + let attempt = remaining.min(per_attempt); + match timeout(attempt, TcpStream::connect(addr)).await { + Ok(Ok(_stream)) => return Ok(()), + Ok(Err(e)) => { + last_error = format!("Cannot reach RocketMQ NameServer {host}: {e}"); + } + Err(_) => { + last_error = + format!("RocketMQ NameServer {host} connect timed out after {}ms", attempt.as_millis()); + } + } + } + } + Err(if last_error.is_empty() { + format!("RocketMQ NameServer connect timed out after {}s", budget.as_secs()) + } else { + last_error + }) +} + fn peeked_message_from_agent_json(idx: usize, message: &serde_json::Value) -> PeekedMessage { let mut properties = HashMap::new(); if let Some(partition) = message.get("partition").and_then(|v| v.as_i64()) { @@ -975,10 +1072,18 @@ mod tests { token_signing: None, connect_override: None, management_connect_override: None, + query_timeout_secs: crate::mq::config::DEFAULT_MQ_QUERY_TIMEOUT_SECS, + connect_timeout_secs: crate::mq::config::DEFAULT_MQ_CONNECT_TIMEOUT_SECS, extra, } } + #[tokio::test] + async fn probe_namesrv_rejects_empty_address_list() { + let err = probe_namesrv_tcp(" ; ; ", Duration::from_millis(200)).await.expect_err("empty"); + assert!(err.contains("empty"), "{err}"); + } + #[test] fn connection_params_map_namesrv_and_acl_credentials() { let cfg = rocketmq_config( @@ -995,6 +1100,7 @@ mod tests { assert_eq!(params.get("cluster_name").and_then(|v| v.as_str()), Some("DefaultCluster")); assert_eq!(params.get("access_key").and_then(|v| v.as_str()), Some("rocket")); assert_eq!(params.get("secret_key").and_then(|v| v.as_str()), Some("secret")); + assert_eq!(params.get("request_timeout_ms").and_then(|v| v.as_u64()), Some(30_000)); } #[test] @@ -1031,6 +1137,26 @@ mod tests { assert_eq!(sub.msg_backlog, 7); } + #[test] + fn rocketmq_subscription_from_group_marks_failed_lag_unavailable() { + let group = serde_json::json!({ + "groupId": "lag-fail", + "groupType": "NORMAL", + "totalLagFailed": true + }); + let sub = rocketmq_subscription_from_group(&group); + assert_eq!(sub.msg_backlog, 0); + assert_eq!(sub.backlog_unavailable, Some(true)); + } + + #[test] + fn rocketmq_subscription_from_group_defaults_missing_type_to_unknown() { + let group = serde_json::json!({ "groupId": "no-type" }); + let sub = rocketmq_subscription_from_group(&group); + assert_eq!(sub.sub_type, "UNKNOWN"); + assert_eq!(sub.consumer_group_type.as_deref(), Some("UNKNOWN")); + } + #[test] fn rocketmq_subscription_from_group_maps_offline_topic_consumer() { let group = serde_json::json!({ diff --git a/crates/dbx-core/src/mq/config.rs b/crates/dbx-core/src/mq/config.rs index 49f5a2b39..beb85d3df 100644 --- a/crates/dbx-core/src/mq/config.rs +++ b/crates/dbx-core/src/mq/config.rs @@ -4,12 +4,19 @@ //! `ConnectionConfig` rather than adding top-level fields, keeping the 50+ //! database-type connection model untouched. +use std::time::Duration; + use serde::{Deserialize, Serialize}; use crate::models::connection::ConnectionConfig; use crate::mq::auth::MqAuth; use crate::mq::types::{MqSystemKind, MqTokenSigningConfig}; +/// Default query timeout when constructing test configs without a ConnectionConfig. +pub const DEFAULT_MQ_QUERY_TIMEOUT_SECS: u64 = 30; +/// Default connect timeout when constructing test configs without a ConnectionConfig. +pub const DEFAULT_MQ_CONNECT_TIMEOUT_SECS: u64 = 10; + /// Runtime TCP endpoint override for an MQ transport. /// /// The logical broker endpoint remains unchanged so TLS hostname verification, @@ -53,6 +60,12 @@ pub struct MqAdminConfig { /// its Management HTTP API listens on an independently configured port. #[serde(skip)] pub management_connect_override: Option, + /// Runtime-only: from `ConnectionConfig.query_timeout_secs` (`0` = unlimited). + #[serde(skip)] + pub query_timeout_secs: u64, + /// Runtime-only: from `ConnectionConfig.effective_connect_timeout_secs()`. + #[serde(skip)] + pub connect_timeout_secs: u64, /// System-specific extension fields (e.g. Kafka bootstrap servers). #[serde(default, skip_serializing_if = "serde_json::Value::is_null")] pub extra: serde_json::Value, @@ -76,9 +89,35 @@ impl MqAdminConfig { { return Err("Message queue admin URL is empty".to_string()); } + // Advanced connection timeouts live on ConnectionConfig, not external_config. + parsed.query_timeout_secs = cfg.effective_query_timeout_secs(); + parsed.connect_timeout_secs = cfg.effective_connect_timeout_secs(); Ok(parsed) } + /// Agent / HTTP RPC wall-clock timeout. `None` disables the client-side timeout. + pub fn rpc_timeout(&self) -> Option { + if self.query_timeout_secs == 0 { + None + } else { + Some(Duration::from_secs(self.query_timeout_secs.max(1))) + } + } + + /// Milliseconds for agent `request_timeout_ms`. Unlimited query timeout maps to a + /// large but finite admin request budget so Java/Go clients still make progress. + pub fn request_timeout_ms(&self) -> u64 { + match self.query_timeout_secs { + 0 => 3_600_000, + secs => secs.saturating_mul(1000).max(1_000), + } + } + + /// Budget for establishing the MQ adapter (agent spawn + handshake/connect). + pub fn connect_timeout(&self) -> Duration { + Duration::from_secs(self.connect_timeout_secs.max(1)) + } + pub fn token_signing_configured(&self) -> bool { self.token_signing.as_ref().is_some_and(MqTokenSigningConfig::is_configured) } @@ -228,6 +267,25 @@ mod tests { assert_eq!(mqc.system_kind, MqSystemKind::RocketMq); assert_eq!(mqc.admin_url, ""); assert_eq!(mqc.extra.get("namesrvAddr").and_then(|v| v.as_str()), Some("127.0.0.1:9876")); + assert_eq!(mqc.query_timeout_secs, 30); + assert_eq!(mqc.connect_timeout_secs, 5); + assert_eq!(mqc.request_timeout_ms(), 30_000); + } + + #[test] + fn copies_advanced_timeouts_from_connection_config() { + let mut cfg = connection_with_external(serde_json::json!({ + "systemKind": "rocketmq", + "adminUrl": "", + "extra": { "namesrvAddr": "127.0.0.1:9876" } + })); + cfg.query_timeout_secs = 120; + cfg.connect_timeout_secs = 15; + let mqc = MqAdminConfig::from_connection(&cfg).expect("parse"); + assert_eq!(mqc.query_timeout_secs, 120); + assert_eq!(mqc.connect_timeout_secs, 15); + assert_eq!(mqc.request_timeout_ms(), 120_000); + assert_eq!(mqc.rpc_timeout(), Some(std::time::Duration::from_secs(120))); } #[test] diff --git a/crates/dbx-core/src/mq/mod.rs b/crates/dbx-core/src/mq/mod.rs index 5d7b7356b..648a6b8f7 100644 --- a/crates/dbx-core/src/mq/mod.rs +++ b/crates/dbx-core/src/mq/mod.rs @@ -45,10 +45,11 @@ pub use crate::mq::types::*; /// Each connection has its own build-lock so concurrent first-use requests for /// the same connection block until the first builder finishes, rather than both /// racing to construct an adapter. -#[derive(Default)] +/// Clone shares the same cache (needed for keepalive tasks that must drop adapters). +#[derive(Clone, Default)] pub struct MqAdminRegistry { - instances: RwLock>, - build_locks: RwLock>>>, + instances: Arc>>, + build_locks: Arc>>>>, } struct CachedMqAdmin { @@ -65,7 +66,7 @@ pub struct MqBuildResult { impl MqAdminRegistry { pub fn new() -> Self { - Self { instances: RwLock::new(HashMap::new()), build_locks: RwLock::new(HashMap::new()) } + Self { instances: Arc::new(RwLock::new(HashMap::new())), build_locks: Arc::new(RwLock::new(HashMap::new())) } } /// Return the cached adapter for this connection, building it from the @@ -108,7 +109,7 @@ impl MqAdminRegistry { // Config changed — drop the stale adapter so its agent process is released. self.instances.write().await.remove(connection_id); - let adapter = build_adapter(mqc, agent_launch).await?; + let adapter = build_adapter_with_connect_timeout(mqc, agent_launch).await?; self.instances .write() .await @@ -116,6 +117,17 @@ impl MqAdminRegistry { Ok(MqBuildResult { adapter, was_cached: false }) } + /// Cached adapter for keepalive / diagnostics. Returns `None` when not built yet. + pub async fn get_cached_adapter(&self, connection_id: &str) -> Option> { + self.instances.read().await.get(connection_id).map(|entry| entry.adapter.clone()) + } + + /// Whether `adapter` is still the live registry entry for this connection (Arc identity). + /// Used so a stale keepalive cannot drop a replacement built after reconnect. + pub async fn is_current_adapter(&self, connection_id: &str, adapter: &Arc) -> bool { + self.instances.read().await.get(connection_id).is_some_and(|entry| Arc::ptr_eq(&entry.adapter, adapter)) + } + /// Drop the cached adapter for a connection (called on disconnect). pub async fn drop_connection(&self, connection_id: &str) { self.instances.write().await.remove(connection_id); @@ -144,7 +156,26 @@ impl MqAdminRegistry { mqc: MqAdminConfig, agent_launch: Option, ) -> Result, String> { - build_adapter(mqc, agent_launch).await + build_adapter_with_connect_timeout(mqc, agent_launch).await + } +} + +async fn build_adapter_with_connect_timeout( + mqc: MqAdminConfig, + agent_launch: Option, +) -> Result, String> { + let budget = mqc.connect_timeout(); + // RocketMQ: TCP-probe NameServer outside the connect wall so cold JVM spawn + // retains the full connect_timeout (probe used to steal up to half of it). + if mqc.system_kind == MqSystemKindInternal::RocketMq { + // Probe runs outside the connect wall; scale with connect_timeout so HA NameServer + // lists are not starved by a hard 2s cap, but keep an upper bound for snappy UX. + let probe_budget = (budget / 2).clamp(std::time::Duration::from_millis(500), std::time::Duration::from_secs(5)); + crate::mq::adapters::rocketmq::probe_namesrv_before_connect(&mqc, probe_budget).await?; + } + match tokio::time::timeout(budget, build_adapter(mqc, agent_launch)).await { + Ok(result) => result, + Err(_) => Err(format!("Message queue connect timed out after {}s", budget.as_secs())), } } diff --git a/crates/dbx-core/src/mq/port.rs b/crates/dbx-core/src/mq/port.rs index c9dedb178..dbc8c7866 100644 --- a/crates/dbx-core/src/mq/port.rs +++ b/crates/dbx-core/src/mq/port.rs @@ -149,6 +149,13 @@ pub trait MessageQueueAdmin: Send + Sync { // ---- Subscriptions ---- async fn list_subscriptions(&self, topic: &TopicRef) -> Result, String>; + + /// Second-pass enrichment for subscription rows (e.g. RocketMQ online members/topics). + /// Default reuses [`list_subscriptions`]; RocketMQ overrides with `enrich: true`. + async fn enrich_subscriptions(&self, topic: &TopicRef) -> Result, String> { + self.list_subscriptions(topic).await + } + async fn create_subscription(&self, topic: &TopicRef, sub: &str, pos: ResetPosition) -> Result<(), String>; async fn delete_subscription(&self, topic: &TopicRef, sub: &str, force: bool) -> Result<(), String>; async fn skip_messages(&self, topic: &TopicRef, sub: &str, count: SkipCount) -> Result<(), String>; diff --git a/crates/dbx-core/src/mq/service.rs b/crates/dbx-core/src/mq/service.rs index 59b1e544d..897b99821 100644 --- a/crates/dbx-core/src/mq/service.rs +++ b/crates/dbx-core/src/mq/service.rs @@ -390,6 +390,15 @@ pub async fn mq_list_subscriptions_core( adapter.list_subscriptions(&topic).await } +pub async fn mq_enrich_subscriptions_core( + state: &AppState, + conn_id: &str, + topic: TopicRef, +) -> Result, String> { + let adapter = get_adapter(state, conn_id).await?; + adapter.enrich_subscriptions(&topic).await +} + pub async fn mq_create_subscription_core( state: &AppState, conn_id: &str, @@ -842,6 +851,8 @@ async fn ensure_connection_writable(state: &AppState, conn_id: &str, operation: config.name, operation )); } + // Production protection for desktop MQ uses UI confirmation; MCP enforces + // is_production separately. Do not hard-block confirmed desktop writes here. } Ok(()) } @@ -939,6 +950,29 @@ mod tests { (state, dir) } + #[tokio::test] + async fn mutating_rocketmq_send_blocks_read_only_connections() { + let (state, _dir) = test_state_with(mq_connection(true)).await; + let err = mq_send_message_core( + &state, + "readonly-mq", + SendMessageRequest { + topic: "t".into(), + key: None, + payload_base64: "eA==".into(), + payload_text: Some("x".into()), + headers: Default::default(), + partition: None, + exchange: None, + routing_key: None, + namespace: None, + }, + ) + .await + .expect_err("read-only must block send"); + assert!(err.contains("Read-only mode"), "{err}"); + } + #[tokio::test] async fn mutating_service_calls_block_read_only_connections_before_adapter_build() { let (state, dir) = test_state_with(mq_connection(true)).await; diff --git a/crates/dbx-core/src/mq/types.rs b/crates/dbx-core/src/mq/types.rs index 278cfae38..11430d6f5 100644 --- a/crates/dbx-core/src/mq/types.rs +++ b/crates/dbx-core/src/mq/types.rs @@ -415,6 +415,9 @@ pub struct SubscriptionInfo { /// RocketMQ: CLUSTERING / BROADCASTING. #[serde(default, skip_serializing_if = "Option::is_none")] pub message_model: Option, + /// When true, backlog probe failed — UI must not treat `msg_backlog` as healthy zero. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backlog_unavailable: Option, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -465,11 +468,76 @@ pub enum SkipCount { Count { count: u32 }, } +/// Per-queue/partition consume progress (RocketMQ Dashboard consume-detail / Kafka lag rows). +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PartitionBacklog { + pub partition: i32, + /// Consumer committed offset (`consumerOffset` / `currentOffset`). + pub current_offset: i64, + /// Broker max offset (`brokerOffset` / `endOffset`). + pub end_offset: i64, + pub lag: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub broker_name: Option, + /// Last consume message store timestamp (ms). `0` means unavailable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_timestamp: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub consumer_client: Option, +} + #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BacklogStats { pub msg_backlog: i64, pub backlog_size: i64, + /// Optional queue-level progress; empty for adapters that only expose totals. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub partitions: Vec, +} + +/// Parse agent `mq_get_consumer_lag` JSON (`totalLag` + `partitions[]`) into [`BacklogStats`]. +pub fn backlog_stats_from_consumer_lag(value: &serde_json::Value) -> BacklogStats { + let msg_backlog = value.get("totalLag").and_then(|v| v.as_i64()).unwrap_or(0); + let partitions = value + .get("partitions") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|row| { + let partition = row.get("partition").and_then(|v| v.as_i64()).map(|v| v as i32)?; + let current_offset = row.get("currentOffset").and_then(|v| v.as_i64()).unwrap_or(0); + let end_offset = row.get("endOffset").and_then(|v| v.as_i64()).unwrap_or(0); + let lag = + row.get("lag").and_then(|v| v.as_i64()).unwrap_or_else(|| (end_offset - current_offset).max(0)); + let broker_name = row + .get("brokerName") + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string); + let last_timestamp = row.get("lastTimestamp").and_then(|v| v.as_i64()); + let consumer_client = row + .get("consumerClient") + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string); + Some(PartitionBacklog { + partition, + current_offset, + end_offset, + lag, + broker_name, + last_timestamp, + consumer_client, + }) + }) + .collect() + }) + .unwrap_or_default(); + BacklogStats { msg_backlog, backlog_size: 0, partitions } } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -1250,4 +1318,52 @@ mod tests { assert!(!legacy.incomplete); assert!(legacy.messages.is_empty()); } + + #[test] + fn backlog_stats_from_consumer_lag_maps_dashboard_fields() { + let lag = serde_json::json!({ + "totalLag": 10, + "partitions": [ + { + "partition": 0, + "currentOffset": 90, + "endOffset": 100, + "lag": 10, + "brokerName": "broker-a", + "lastTimestamp": 1725000000000_i64, + "consumerClient": "172.18.2.212@7#1" + }, + { + "partition": 1, + "currentOffset": 50, + "endOffset": 50, + "lag": 0, + "brokerName": "", + "lastTimestamp": 0, + "consumerClient": "" + } + ] + }); + let stats = super::backlog_stats_from_consumer_lag(&lag); + assert_eq!(stats.msg_backlog, 10); + assert_eq!(stats.partitions.len(), 2); + assert_eq!(stats.partitions[0].broker_name.as_deref(), Some("broker-a")); + assert_eq!(stats.partitions[0].consumer_client.as_deref(), Some("172.18.2.212@7#1")); + assert_eq!(stats.partitions[0].last_timestamp, Some(1725000000000)); + // Empty strings are normalized to None so UI can show "-". + assert!(stats.partitions[1].broker_name.is_none()); + assert!(stats.partitions[1].consumer_client.is_none()); + assert_eq!(stats.partitions[1].last_timestamp, Some(0)); + + let json = serde_json::to_value(&stats).expect("serialize backlog stats"); + assert_eq!(json.get("msgBacklog").and_then(|v| v.as_i64()), Some(10)); + assert!(json.get("partitions").and_then(|v| v.as_array()).is_some()); + + // Legacy callers that only had totals still deserialize with empty partitions. + let legacy: super::BacklogStats = + serde_json::from_value(serde_json::json!({ "msgBacklog": 3, "backlogSize": 0 })) + .expect("legacy backlog stats"); + assert_eq!(legacy.msg_backlog, 3); + assert!(legacy.partitions.is_empty()); + } } diff --git a/crates/dbx-web/src/main.rs b/crates/dbx-web/src/main.rs index 8cff8f1cc..da5ca6d97 100644 --- a/crates/dbx-web/src/main.rs +++ b/crates/dbx-web/src/main.rs @@ -134,6 +134,7 @@ fn add_mq_routes(router: Router>) -> Router> { .route("/mq/messages/query-by-topic", post(routes::mq::query_messages_by_topic)) .route("/mq/messages/trace", post(routes::mq::query_message_trace)) .route("/mq/subscriptions/list", post(routes::mq::list_subscriptions)) + .route("/mq/subscriptions/enrich", post(routes::mq::enrich_subscriptions)) .route("/mq/subscriptions/create", post(routes::mq::create_subscription)) .route("/mq/subscriptions/delete", post(routes::mq::delete_subscription)) .route("/mq/subscriptions/skip-messages", post(routes::mq::skip_messages)) diff --git a/crates/dbx-web/src/routes/mq.rs b/crates/dbx-web/src/routes/mq.rs index e19784d6c..de792d4a1 100644 --- a/crates/dbx-web/src/routes/mq.rs +++ b/crates/dbx-web/src/routes/mq.rs @@ -751,6 +751,18 @@ pub async fn list_subscriptions( Ok(Json(result)) } +pub async fn enrich_subscriptions( + State(state): State>, + headers: HeaderMap, + Json(req): Json, +) -> Result>, AppError> { + super::mcp_policy::ensure_scope(&state, &headers, &req.connection_id).await?; + let result = dbx_core::mq::service::mq_enrich_subscriptions_core(&state.app, &req.connection_id, req.topic) + .await + .map_err(AppError::from)?; + Ok(Json(result)) +} + pub async fn create_subscription( State(state): State>, headers: HeaderMap, diff --git a/src-tauri/src/commands/mq_cmd.rs b/src-tauri/src/commands/mq_cmd.rs index ced221d49..06d9493d5 100644 --- a/src-tauri/src/commands/mq_cmd.rs +++ b/src-tauri/src/commands/mq_cmd.rs @@ -267,6 +267,15 @@ pub async fn mq_list_subscriptions( dbx_core::mq::service::mq_list_subscriptions_core(&state, &connection_id, topic).await } +#[tauri::command] +pub async fn mq_enrich_subscriptions( + state: State<'_, Arc>, + connection_id: String, + topic: dbx_core::mq::TopicRef, +) -> Result, String> { + dbx_core::mq::service::mq_enrich_subscriptions_core(&state, &connection_id, topic).await +} + #[tauri::command] pub async fn mq_create_subscription( state: State<'_, Arc>, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index aa47dd801..f2c35b99c 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1873,6 +1873,8 @@ pub fn run() { #[cfg(feature = "mq-admin")] commands::mq_cmd::mq_list_subscriptions, #[cfg(feature = "mq-admin")] + commands::mq_cmd::mq_enrich_subscriptions, + #[cfg(feature = "mq-admin")] commands::mq_cmd::mq_create_subscription, #[cfg(feature = "mq-admin")] commands::mq_cmd::mq_delete_subscription,