fix(core): reconnect stale agent pools on JDBC errors

This commit is contained in:
zipg 2026-07-09 18:27:23 +08:00 committed by GitHub
parent 24370135eb
commit e25be6335d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 81 additions and 12 deletions

View File

@ -0,0 +1,15 @@
import { describe, expect, it } from "vitest";
import { shouldMarkDisconnected } from "@/lib/connection/connectionHealth";
describe("connectionHealth", () => {
it("marks localized and JDBC communication errors as disconnected", () => {
expect(shouldMarkDisconnected("Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常")).toBe(true);
expect(shouldMarkDisconnected("Agent RPC error (-1): com.mysql.cj.jdbc.exceptions.CommunicationsException: Communications link failure")).toBe(true);
expect(shouldMarkDisconnected("Agent RPC error (-1): java.sql.SQLRecoverableException: IO 错误: Got minus one from a read call")).toBe(true);
});
it("does not mark ordinary SQL or authentication errors as disconnected", () => {
expect(shouldMarkDisconnected("syntax error at or near SELECT")).toBe(false);
expect(shouldMarkDisconnected("Access denied for user root")).toBe(false);
});
});

View File

@ -24,6 +24,12 @@ const CONNECTION_ERROR_PATTERNS = [
"input/output error",
"关闭的连接",
"连接已关闭",
"网络通信异常",
"通信异常",
"communications link failure",
"sqlrecoverableexception",
"sqlnontransientconnectionexception",
"sqltransientconnectionexception",
"i/o error",
"no route to host",
];

View File

@ -3,7 +3,7 @@ use std::io::{BufRead, BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command, Stdio};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::time::{Duration, Instant};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
@ -14,7 +14,8 @@ pub const AGENT_PROTOCOL_VERSION: u32 = 1;
const RPC_TIMEOUT_SECS: u64 = 30;
const STARTUP_TIMEOUT_SECS: u64 = 15;
const STDERR_TAIL_LINES: usize = 20;
const AGENT_EXIT_DIAGNOSTIC_WAIT_MS: u64 = 200;
const AGENT_EXIT_DIAGNOSTIC_WAIT_MS: u64 = 1_000;
const AGENT_EXIT_DIAGNOSTIC_POLL_MS: u64 = 10;
const AGENT_JAVA_OPTS_ENV: &str = "DBX_AGENT_JAVA_OPTS";
const AGENT_JAVA_TOO_OLD_MESSAGE: &str =
"Agent requires Java 21, but DBX started it with an older Java runtime. Use DBX managed JRE 21 or select a Java 21 executable in Driver Manager.";
@ -1378,12 +1379,16 @@ fn child_exit_status(child: &mut Child) -> Option<String> {
}
fn child_exit_status_after_short_wait(child: &mut Child) -> Option<String> {
let status = child_exit_status(child);
if status.is_some() {
return status;
let deadline = Instant::now() + Duration::from_millis(AGENT_EXIT_DIAGNOSTIC_WAIT_MS);
loop {
if let Some(status) = child_exit_status(child) {
return Some(status);
}
if Instant::now() >= deadline {
return None;
}
std::thread::sleep(Duration::from_millis(AGENT_EXIT_DIAGNOSTIC_POLL_MS));
}
std::thread::sleep(Duration::from_millis(AGENT_EXIT_DIAGNOSTIC_WAIT_MS));
child_exit_status(child)
}
fn stderr_tail_snapshot(stderr_tail: &Arc<Mutex<StderrTail>>) -> StderrTail {
@ -1429,11 +1434,9 @@ fn format_agent_startup_error(base: &str, child: &mut Child, stderr_tail: &Arc<M
impl AgentDriverClient {
fn format_agent_process_error(&mut self, base: &str) -> String {
format_agent_process_error(
base,
child_exit_status_after_short_wait(&mut self.child),
&stderr_tail_snapshot(&self.stderr_tail),
)
// Runtime RPC errors are common SQL/driver paths. Do not wait for the
// child to exit unless startup diagnostics already expect the process to die.
format_agent_process_error(base, child_exit_status(&mut self.child), &stderr_tail_snapshot(&self.stderr_tail))
}
}
@ -1622,6 +1625,33 @@ mod tests {
assert!(message.contains("UnsupportedClassVersionError"));
}
#[test]
fn runtime_agent_process_error_does_not_wait_for_live_child() {
let child = Command::new("sh")
.arg("-c")
.arg("sleep 2")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("child should start");
let mut client = AgentDriverClient {
child,
stdin: None,
stdout: None,
stderr_tail: Arc::new(Mutex::new(StderrTail::default())),
handshake: None,
next_id: 0,
};
let started_at = std::time::Instant::now();
let message = client.format_agent_process_error("Agent RPC error (-1): syntax error");
assert!(started_at.elapsed() < std::time::Duration::from_millis(500));
assert!(message.contains("Agent RPC error (-1): syntax error"));
assert!(!message.contains("agent process exited"));
}
#[test]
fn stderr_tail_keeps_recent_lines_only() {
let mut stderr_tail = StderrTail::with_capacity(3);

View File

@ -745,8 +745,15 @@ pub fn is_connection_error(err: &str) -> bool {
|| lower.contains("closed")
|| lower.contains("关闭的连接")
|| lower.contains("连接已关闭")
|| lower.contains("网络通信异常")
|| lower.contains("通信异常")
|| lower.contains("communications link failure")
|| lower.contains("sqlrecoverableexception")
|| lower.contains("sqlnontransientconnectionexception")
|| lower.contains("sqltransientconnectionexception")
|| lower.contains("eof")
|| lower.contains("i/o error")
|| lower.contains("input/output error")
|| lower.contains("not connected")
|| lower.contains("end-of-file")
|| lower.contains("idle")
@ -3296,6 +3303,13 @@ mod tests {
assert!(is_connection_error(
"I/O error: 由于连接方在一段时间后没有正确答复或连接的主机没有反应,连接尝试失败。 (os error 10060)"
));
assert!(is_connection_error("Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常"));
assert!(is_connection_error(
"Agent RPC error (-1): java.sql.SQLRecoverableException: IO 错误: Got minus one from a read call"
));
assert!(is_connection_error(
"Agent RPC error (-1): com.mysql.cj.jdbc.exceptions.CommunicationsException: Communications link failure"
));
}
#[test]
@ -3365,6 +3379,9 @@ mod tests {
assert_eq!(pool_error_action(Some(DatabaseType::SqlServer), err), PoolErrorAction::ReconnectAndRetry);
assert_eq!(pool_error_action(Some(DatabaseType::Postgres), err), PoolErrorAction::ReconnectAndRetry);
let dameng_err = "Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常";
assert_eq!(pool_error_action(Some(DatabaseType::Dameng), dameng_err), PoolErrorAction::ReconnectAndRetry);
}
#[cfg(feature = "duckdb-bundled")]

View File

@ -2357,6 +2357,7 @@ mod tests {
fn metadata_retry_recovers_missing_pool_only_as_transient_state() {
assert!(is_retryable_metadata_error("Pool not found"));
assert!(is_retryable_metadata_error("connection reset by peer"));
assert!(is_retryable_metadata_error("Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常"));
assert!(!is_retryable_metadata_error("Unknown column 'email' in 'field list'"));
assert!(!is_retryable_metadata_error("Access denied for user"));
}