fix(core): reconnect stale agent pools on JDBC errors
This commit is contained in:
parent
24370135eb
commit
e25be6335d
|
|
@ -0,0 +1,15 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { shouldMarkDisconnected } from "@/lib/connection/connectionHealth";
|
||||
|
||||
describe("connectionHealth", () => {
|
||||
it("marks localized and JDBC communication errors as disconnected", () => {
|
||||
expect(shouldMarkDisconnected("Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常")).toBe(true);
|
||||
expect(shouldMarkDisconnected("Agent RPC error (-1): com.mysql.cj.jdbc.exceptions.CommunicationsException: Communications link failure")).toBe(true);
|
||||
expect(shouldMarkDisconnected("Agent RPC error (-1): java.sql.SQLRecoverableException: IO 错误: Got minus one from a read call")).toBe(true);
|
||||
});
|
||||
|
||||
it("does not mark ordinary SQL or authentication errors as disconnected", () => {
|
||||
expect(shouldMarkDisconnected("syntax error at or near SELECT")).toBe(false);
|
||||
expect(shouldMarkDisconnected("Access denied for user root")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -24,6 +24,12 @@ const CONNECTION_ERROR_PATTERNS = [
|
|||
"input/output error",
|
||||
"关闭的连接",
|
||||
"连接已关闭",
|
||||
"网络通信异常",
|
||||
"通信异常",
|
||||
"communications link failure",
|
||||
"sqlrecoverableexception",
|
||||
"sqlnontransientconnectionexception",
|
||||
"sqltransientconnectionexception",
|
||||
"i/o error",
|
||||
"no route to host",
|
||||
];
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ use std::io::{BufRead, BufReader, BufWriter, Write};
|
|||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command, Stdio};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
@ -14,7 +14,8 @@ pub const AGENT_PROTOCOL_VERSION: u32 = 1;
|
|||
const RPC_TIMEOUT_SECS: u64 = 30;
|
||||
const STARTUP_TIMEOUT_SECS: u64 = 15;
|
||||
const STDERR_TAIL_LINES: usize = 20;
|
||||
const AGENT_EXIT_DIAGNOSTIC_WAIT_MS: u64 = 200;
|
||||
const AGENT_EXIT_DIAGNOSTIC_WAIT_MS: u64 = 1_000;
|
||||
const AGENT_EXIT_DIAGNOSTIC_POLL_MS: u64 = 10;
|
||||
const AGENT_JAVA_OPTS_ENV: &str = "DBX_AGENT_JAVA_OPTS";
|
||||
const AGENT_JAVA_TOO_OLD_MESSAGE: &str =
|
||||
"Agent requires Java 21, but DBX started it with an older Java runtime. Use DBX managed JRE 21 or select a Java 21 executable in Driver Manager.";
|
||||
|
|
@ -1378,12 +1379,16 @@ fn child_exit_status(child: &mut Child) -> Option<String> {
|
|||
}
|
||||
|
||||
fn child_exit_status_after_short_wait(child: &mut Child) -> Option<String> {
|
||||
let status = child_exit_status(child);
|
||||
if status.is_some() {
|
||||
return status;
|
||||
let deadline = Instant::now() + Duration::from_millis(AGENT_EXIT_DIAGNOSTIC_WAIT_MS);
|
||||
loop {
|
||||
if let Some(status) = child_exit_status(child) {
|
||||
return Some(status);
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return None;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(AGENT_EXIT_DIAGNOSTIC_POLL_MS));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(AGENT_EXIT_DIAGNOSTIC_WAIT_MS));
|
||||
child_exit_status(child)
|
||||
}
|
||||
|
||||
fn stderr_tail_snapshot(stderr_tail: &Arc<Mutex<StderrTail>>) -> StderrTail {
|
||||
|
|
@ -1429,11 +1434,9 @@ fn format_agent_startup_error(base: &str, child: &mut Child, stderr_tail: &Arc<M
|
|||
|
||||
impl AgentDriverClient {
|
||||
fn format_agent_process_error(&mut self, base: &str) -> String {
|
||||
format_agent_process_error(
|
||||
base,
|
||||
child_exit_status_after_short_wait(&mut self.child),
|
||||
&stderr_tail_snapshot(&self.stderr_tail),
|
||||
)
|
||||
// Runtime RPC errors are common SQL/driver paths. Do not wait for the
|
||||
// child to exit unless startup diagnostics already expect the process to die.
|
||||
format_agent_process_error(base, child_exit_status(&mut self.child), &stderr_tail_snapshot(&self.stderr_tail))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1622,6 +1625,33 @@ mod tests {
|
|||
assert!(message.contains("UnsupportedClassVersionError"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_agent_process_error_does_not_wait_for_live_child() {
|
||||
let child = Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg("sleep 2")
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.expect("child should start");
|
||||
let mut client = AgentDriverClient {
|
||||
child,
|
||||
stdin: None,
|
||||
stdout: None,
|
||||
stderr_tail: Arc::new(Mutex::new(StderrTail::default())),
|
||||
handshake: None,
|
||||
next_id: 0,
|
||||
};
|
||||
|
||||
let started_at = std::time::Instant::now();
|
||||
let message = client.format_agent_process_error("Agent RPC error (-1): syntax error");
|
||||
|
||||
assert!(started_at.elapsed() < std::time::Duration::from_millis(500));
|
||||
assert!(message.contains("Agent RPC error (-1): syntax error"));
|
||||
assert!(!message.contains("agent process exited"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stderr_tail_keeps_recent_lines_only() {
|
||||
let mut stderr_tail = StderrTail::with_capacity(3);
|
||||
|
|
|
|||
|
|
@ -745,8 +745,15 @@ pub fn is_connection_error(err: &str) -> bool {
|
|||
|| lower.contains("closed")
|
||||
|| lower.contains("关闭的连接")
|
||||
|| lower.contains("连接已关闭")
|
||||
|| lower.contains("网络通信异常")
|
||||
|| lower.contains("通信异常")
|
||||
|| lower.contains("communications link failure")
|
||||
|| lower.contains("sqlrecoverableexception")
|
||||
|| lower.contains("sqlnontransientconnectionexception")
|
||||
|| lower.contains("sqltransientconnectionexception")
|
||||
|| lower.contains("eof")
|
||||
|| lower.contains("i/o error")
|
||||
|| lower.contains("input/output error")
|
||||
|| lower.contains("not connected")
|
||||
|| lower.contains("end-of-file")
|
||||
|| lower.contains("idle")
|
||||
|
|
@ -3296,6 +3303,13 @@ mod tests {
|
|||
assert!(is_connection_error(
|
||||
"I/O error: 由于连接方在一段时间后没有正确答复或连接的主机没有反应,连接尝试失败。 (os error 10060)"
|
||||
));
|
||||
assert!(is_connection_error("Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常"));
|
||||
assert!(is_connection_error(
|
||||
"Agent RPC error (-1): java.sql.SQLRecoverableException: IO 错误: Got minus one from a read call"
|
||||
));
|
||||
assert!(is_connection_error(
|
||||
"Agent RPC error (-1): com.mysql.cj.jdbc.exceptions.CommunicationsException: Communications link failure"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -3365,6 +3379,9 @@ mod tests {
|
|||
|
||||
assert_eq!(pool_error_action(Some(DatabaseType::SqlServer), err), PoolErrorAction::ReconnectAndRetry);
|
||||
assert_eq!(pool_error_action(Some(DatabaseType::Postgres), err), PoolErrorAction::ReconnectAndRetry);
|
||||
|
||||
let dameng_err = "Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常";
|
||||
assert_eq!(pool_error_action(Some(DatabaseType::Dameng), dameng_err), PoolErrorAction::ReconnectAndRetry);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
|
|
|
|||
|
|
@ -2357,6 +2357,7 @@ mod tests {
|
|||
fn metadata_retry_recovers_missing_pool_only_as_transient_state() {
|
||||
assert!(is_retryable_metadata_error("Pool not found"));
|
||||
assert!(is_retryable_metadata_error("connection reset by peer"));
|
||||
assert!(is_retryable_metadata_error("Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常"));
|
||||
assert!(!is_retryable_metadata_error("Unknown column 'email' in 'field list'"));
|
||||
assert!(!is_retryable_metadata_error("Access denied for user"));
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue