feat(lib/proxy): support multiple proxier by `PROXY_URIS` env

This commit is contained in:
pseudoyu 2025-06-24 11:05:14 +08:00
parent 18c3409322
commit 3d9ac5c6ee
No known key found for this signature in database
6 changed files with 326 additions and 40 deletions

View File

@ -37,6 +37,7 @@ export type Config = {
};
// proxy
proxyUri?: string;
proxyUris?: string[];
proxy: {
protocol?: string;
host?: string;
@ -44,6 +45,8 @@ export type Config = {
auth?: string;
url_regex: string;
strategy: 'on_retry' | 'all';
failoverTimeout?: number;
healthCheckInterval?: number;
};
pacUri?: string;
pacScript?: string;
@ -491,6 +494,11 @@ const calculateValue = () => {
},
// proxy
proxyUri: envs.PROXY_URI,
proxyUris: envs.PROXY_URIS
? envs.PROXY_URIS.split(',')
.map((uri) => uri.trim())
.filter(Boolean)
: undefined,
proxy: {
protocol: envs.PROXY_PROTOCOL,
host: envs.PROXY_HOST,
@ -498,6 +506,8 @@ const calculateValue = () => {
auth: envs.PROXY_AUTH,
url_regex: envs.PROXY_URL_REGEX || '.*',
strategy: envs.PROXY_STRATEGY || 'all', // all / on_retry
failoverTimeout: toInt(envs.PROXY_FAILOVER_TIMEOUT, 5000),
healthCheckInterval: toInt(envs.PROXY_HEALTH_CHECK_INTERVAL, 60000),
},
pacUri: envs.PAC_URI,
pacScript: envs.PAC_SCRIPT,

View File

@ -3,20 +3,72 @@ import { PacProxyAgent } from 'pac-proxy-agent';
import { HttpsProxyAgent } from 'https-proxy-agent';
import { SocksProxyAgent } from 'socks-proxy-agent';
import { ProxyAgent } from 'undici';
import logger from '@/utils/logger';
const proxyIsPAC = config.pacUri || config.pacScript;
import pacProxy from './pac-proxy';
import unifyProxy from './unify-proxy';
import createMultiProxy, { type MultiProxyResult, type ProxyState } from './multi-proxy';
interface ProxyExport {
agent: PacProxyAgent<string> | HttpsProxyAgent<string> | SocksProxyAgent | null;
dispatcher: ProxyAgent | null;
proxyUri?: string;
proxyObj: Record<string, any>;
proxyUrlHandler?: URL | null;
multiProxy?: MultiProxyResult;
getCurrentProxy: () => ProxyState | null;
markProxyFailed: (proxyUri: string) => void;
getAgentForProxy: (proxyState: ProxyState) => any;
getDispatcherForProxy: (proxyState: ProxyState) => ProxyAgent | null;
}
let proxyUri: string | undefined;
let proxyObj: Record<string, any> | undefined;
let proxyObj: Record<string, any> = {};
let proxyUrlHandler: URL | null = null;
let multiProxy: MultiProxyResult | undefined;
const createAgentForProxy = (uri: string, proxyObj: Record<string, any>): any => {
if (uri.startsWith('http')) {
return new HttpsProxyAgent(uri, {
headers: {
'proxy-authorization': proxyObj?.auth ? `Basic ${proxyObj.auth}` : undefined,
},
});
} else if (uri.startsWith('socks')) {
return new SocksProxyAgent(uri);
}
return null;
};
const createDispatcherForProxy = (uri: string, proxyObj: Record<string, any>): ProxyAgent | null => {
if (uri.startsWith('http')) {
return new ProxyAgent({
uri,
token: proxyObj?.auth ? `Basic ${proxyObj.auth}` : undefined,
requestTls: {
rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0',
},
});
}
return null;
};
if (proxyIsPAC) {
const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy);
proxyUri = proxy.proxyUri;
proxyObj = proxy.proxyObj;
proxyUrlHandler = proxy.proxyUrlHandler;
} else if (config.proxyUris && config.proxyUris.length > 0) {
multiProxy = createMultiProxy(config.proxyUris, config.proxy);
proxyObj = multiProxy.proxyObj;
const currentProxy = multiProxy.getNextProxy();
if (currentProxy) {
proxyUri = currentProxy.uri;
proxyUrlHandler = currentProxy.urlHandler;
}
logger.info(`Multi-proxy initialized with ${config.proxyUris.length} proxies`);
} else {
const proxy = unifyProxy(config.proxyUri, config.proxy);
proxyUri = proxy.proxyUri;
@ -26,31 +78,63 @@ if (proxyIsPAC) {
let agent: PacProxyAgent<string> | HttpsProxyAgent<string> | SocksProxyAgent | null = null;
let dispatcher: ProxyAgent | null = null;
if (proxyIsPAC) {
if (proxyIsPAC && proxyUri) {
agent = new PacProxyAgent(`pac+${proxyUri}`);
} else if (proxyUri) {
if (proxyUri.startsWith('http')) {
agent = new HttpsProxyAgent(proxyUri, {
headers: {
'proxy-authorization': config.proxy?.auth ? `Basic ${config.proxy?.auth}` : undefined,
},
});
dispatcher = new ProxyAgent({
uri: proxyUri,
token: config.proxy?.auth ? `Basic ${config.proxy?.auth}` : undefined,
requestTls: {
rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0',
},
});
} else if (proxyUri.startsWith('socks')) {
agent = new SocksProxyAgent(proxyUri);
}
agent = createAgentForProxy(proxyUri, proxyObj);
dispatcher = createDispatcherForProxy(proxyUri, proxyObj);
}
export default {
const getCurrentProxy = (): ProxyState | null => {
if (multiProxy) {
return multiProxy.getNextProxy();
}
if (proxyUri) {
return {
uri: proxyUri,
isActive: true,
failureCount: 0,
urlHandler: proxyUrlHandler,
};
}
return null;
};
const markProxyFailed = (failedProxyUri: string) => {
if (multiProxy) {
multiProxy.markProxyFailed(failedProxyUri);
const nextProxy = multiProxy.getNextProxy();
if (nextProxy) {
proxyUri = nextProxy.uri;
proxyUrlHandler = nextProxy.urlHandler || null;
agent = createAgentForProxy(nextProxy.uri, proxyObj);
dispatcher = createDispatcherForProxy(nextProxy.uri, proxyObj);
logger.info(`Switched to proxy: ${nextProxy.uri}`);
} else {
logger.warn('No available proxies remaining');
agent = null;
dispatcher = null;
proxyUri = undefined;
}
}
};
const getAgentForProxy = (proxyState: ProxyState) => createAgentForProxy(proxyState.uri, proxyObj);
const getDispatcherForProxy = (proxyState: ProxyState) => createDispatcherForProxy(proxyState.uri, proxyObj);
const proxyExport: ProxyExport = {
agent,
dispatcher,
proxyUri,
proxyObj,
proxyUrlHandler,
multiProxy,
getCurrentProxy,
markProxyFailed,
getAgentForProxy,
getDispatcherForProxy,
};
export default proxyExport;

View File

@ -0,0 +1,139 @@
import { type Config } from '@/config';
import logger from '@/utils/logger';
import unifyProxy from './unify-proxy';
export interface ProxyState {
uri: string;
isActive: boolean;
failureCount: number;
lastFailureTime?: number;
agent?: any;
dispatcher?: any;
urlHandler?: URL | null;
}
export interface MultiProxyResult {
currentProxy?: ProxyState | null;
allProxies: ProxyState[];
proxyObj: Config['proxy'];
getNextProxy: () => ProxyState | null;
markProxyFailed: (proxyUri: string) => void;
resetProxy: (proxyUri: string) => void;
}
const createMultiProxy = (proxyUris: string[], proxyObj: Config['proxy']): MultiProxyResult => {
const proxies: ProxyState[] = [];
let currentProxyIndex = 0;
for (const uri of proxyUris) {
const unifiedProxy = unifyProxy(uri, proxyObj);
if (unifiedProxy.proxyUri) {
proxies.push({
uri: unifiedProxy.proxyUri,
isActive: true,
failureCount: 0,
urlHandler: unifiedProxy.proxyUrlHandler,
});
}
}
if (proxies.length === 0) {
logger.warn('No valid proxies found in the provided list');
return {
allProxies: [],
proxyObj: proxyObj || {},
getNextProxy: () => null,
markProxyFailed: () => {},
resetProxy: () => {},
};
}
const healthCheckInterval = proxyObj?.healthCheckInterval || 60000;
const maxFailures = 3;
const healthCheck = () => {
const now = Date.now();
for (const proxy of proxies) {
if (!proxy.isActive && proxy.lastFailureTime && now - proxy.lastFailureTime > healthCheckInterval) {
proxy.isActive = true;
proxy.failureCount = 0;
delete proxy.lastFailureTime;
logger.info(`Proxy ${proxy.uri} marked as active again after health check`);
}
}
};
setInterval(healthCheck, healthCheckInterval);
const getNextProxy = (): ProxyState | null => {
const activeProxies = proxies.filter((p) => p.isActive);
if (activeProxies.length === 0) {
logger.warn('No active proxies available');
return null;
}
let nextProxy = activeProxies[currentProxyIndex % activeProxies.length];
let attempts = 0;
while (!nextProxy.isActive && attempts < activeProxies.length) {
currentProxyIndex = (currentProxyIndex + 1) % activeProxies.length;
nextProxy = activeProxies[currentProxyIndex];
attempts++;
}
if (!nextProxy.isActive) {
return null;
}
return nextProxy;
};
const markProxyFailed = (proxyUri: string) => {
const proxy = proxies.find((p) => p.uri === proxyUri);
if (proxy) {
proxy.failureCount++;
proxy.lastFailureTime = Date.now();
if (proxy.failureCount >= maxFailures) {
proxy.isActive = false;
logger.warn(`Proxy ${proxyUri} marked as inactive after ${maxFailures} failures`);
} else {
logger.warn(`Proxy ${proxyUri} failed (${proxy.failureCount}/${maxFailures})`);
}
const activeProxies = proxies.filter((p) => p.isActive);
if (activeProxies.length > 0) {
currentProxyIndex = (currentProxyIndex + 1) % activeProxies.length;
const nextProxy = getNextProxy();
if (nextProxy) {
logger.info(`Switching to proxy: ${nextProxy.uri}`);
}
}
}
};
const resetProxy = (proxyUri: string) => {
const proxy = proxies.find((p) => p.uri === proxyUri);
if (proxy) {
proxy.isActive = true;
proxy.failureCount = 0;
delete proxy.lastFailureTime;
logger.info(`Proxy ${proxyUri} manually reset`);
}
};
const currentProxy = getNextProxy();
if (currentProxy) {
logger.info(`Initial proxy selected: ${currentProxy.uri}`);
}
return {
currentProxy,
allProxies: proxies,
proxyObj: proxyObj || {},
getNextProxy,
markProxyFailed,
resetProxy,
};
};
export default createMultiProxy;

View File

@ -4,7 +4,7 @@ import logger from '@/utils/logger';
const defaultProtocol = 'http';
const possibleProtocol = ['http', 'https', 'socks', 'socks4', 'socks4a', 'socks5', 'socks5h'];
const unifyProxy = (proxyUri: Config['proxyUri'], proxyObj: Config['proxy']) => {
const unifyProxy = (proxyUri: Config['proxyUri'] | string, proxyObj: Config['proxy']) => {
proxyObj = proxyObj || {};
const [oriProxyUri, oriProxyObj] = [proxyUri, proxyObj];
proxyObj = { ...proxyObj };
@ -110,4 +110,6 @@ const unifyProxy = (proxyUri: Config['proxyUri'], proxyObj: Config['proxy']) =>
return { proxyUri, proxyObj, proxyUrlHandler };
};
export const unifyProxies = (proxyUris: string[], proxyObj: Config['proxy']) => proxyUris.map((uri) => unifyProxy(uri, proxyObj)).filter((result) => result.proxyUri);
export default unifyProxy;

View File

@ -30,17 +30,18 @@ const outPuppeteer = async (
insidePuppeteer.use(StealthPlugin());
}
if (proxy.proxyUri && proxy.proxyObj.url_regex === '.*') {
if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) {
const currentProxy = proxy.getCurrentProxy();
if (currentProxy && proxy.proxyObj.url_regex === '.*') {
if (currentProxy.urlHandler?.username || currentProxy.urlHandler?.password) {
// only proxies with authentication need to be anonymized
if (proxy.proxyUrlHandler.protocol === 'http:') {
options.args.push(`--proxy-server=${await anonymizeProxy(proxy.proxyUri)}`);
if (currentProxy.urlHandler.protocol === 'http:') {
options.args.push(`--proxy-server=${await anonymizeProxy(currentProxy.uri)}`);
} else {
logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy');
}
} else {
// Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes
options.args.push(`--proxy-server=${proxy.proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
options.args.push(`--proxy-server=${currentProxy.uri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
}
}
const browser = await (config.puppeteerWSEndpoint
@ -101,11 +102,14 @@ export const getPuppeteerPage = async (
}
let hasProxy = false;
if (proxy.proxyUri && allowProxy) {
if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) {
let currentProxyState: any = null;
const currentProxy = proxy.getCurrentProxy();
if (currentProxy && allowProxy) {
currentProxyState = currentProxy;
if (currentProxy.urlHandler?.username || currentProxy.urlHandler?.password) {
// only proxies with authentication need to be anonymized
if (proxy.proxyUrlHandler.protocol === 'http:') {
const urlObj = new URL(proxy.proxyUri);
if (currentProxy.urlHandler.protocol === 'http:') {
const urlObj = new URL(currentProxy.uri);
urlObj.username = '';
urlObj.password = '';
options.args.push(`--proxy-server=${urlObj.toString().replace(/\/$/, '')}`);
@ -115,7 +119,7 @@ export const getPuppeteerPage = async (
}
} else {
// Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes
options.args.push(`--proxy-server=${proxy.proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
options.args.push(`--proxy-server=${currentProxy.uri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
hasProxy = true;
}
}
@ -145,14 +149,14 @@ export const getPuppeteerPage = async (
const page = await browser.newPage();
if (hasProxy) {
logger.debug(`Proxying request in puppeteer: ${url}`);
if (hasProxy && currentProxyState) {
logger.debug(`Proxying request in puppeteer via ${currentProxyState.uri}: ${url}`);
}
if (hasProxy && (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password)) {
if (hasProxy && currentProxyState && (currentProxyState.urlHandler?.username || currentProxyState.urlHandler?.password)) {
await page.authenticate({
username: proxy.proxyUrlHandler?.username,
password: proxy.proxyUrlHandler?.password,
username: currentProxyState.urlHandler?.username,
password: currentProxyState.urlHandler?.password,
});
}
@ -161,7 +165,16 @@ export const getPuppeteerPage = async (
}
if (!instanceOptions.noGoto) {
await page.goto(url, instanceOptions.gotoConfig || { waitUntil: 'domcontentloaded' });
try {
await page.goto(url, instanceOptions.gotoConfig || { waitUntil: 'domcontentloaded' });
} catch (error) {
if (hasProxy && currentProxyState && proxy.multiProxy) {
logger.warn(`Puppeteer navigation failed with proxy ${currentProxyState.uri}, marking as failed: ${error}`);
proxy.markProxyFailed(currentProxyState.uri);
throw error;
}
throw error;
}
}
return {

View File

@ -60,7 +60,7 @@ const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: Requ
config.enableRemoteDebugging && useCustomHeader(request.headers);
// proxy
if (!init?.dispatcher && proxy.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) {
if (!init?.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) {
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
let urlHandler;
try {
@ -70,13 +70,51 @@ const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: Requ
}
if (proxyRegex.test(request.url) && request.url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) {
options.dispatcher = proxy.dispatcher;
logger.debug(`Proxying request: ${request.url}`);
const currentProxy = proxy.getCurrentProxy();
if (currentProxy) {
const dispatcher = proxy.getDispatcherForProxy(currentProxy);
if (dispatcher) {
options.dispatcher = dispatcher;
logger.debug(`Proxying request via ${currentProxy.uri}: ${request.url}`);
}
}
}
}
await limiterQueue.removeTokens(1);
return undici.fetch(request, options);
const maxRetries = proxy.multiProxy?.allProxies.length || 1;
const attemptRequest = async (attempt: number): Promise<Response> => {
try {
return await undici.fetch(request, options);
} catch (error) {
if (options.dispatcher && proxy.multiProxy && attempt < maxRetries - 1) {
const currentProxy = proxy.getCurrentProxy();
if (currentProxy) {
logger.warn(`Request failed with proxy ${currentProxy.uri}, trying next proxy: ${error}`);
proxy.markProxyFailed(currentProxy.uri);
const nextProxy = proxy.getCurrentProxy();
if (nextProxy && nextProxy.uri !== currentProxy.uri) {
const nextDispatcher = proxy.getDispatcherForProxy(nextProxy);
if (nextDispatcher) {
options.dispatcher = nextDispatcher;
}
logger.debug(`Retrying request with proxy ${nextProxy.uri}: ${request.url}`);
return attemptRequest(attempt + 1);
} else {
logger.warn('No more proxies available, trying without proxy');
delete options.dispatcher;
return attemptRequest(attempt + 1);
}
}
}
throw error;
}
};
return attemptRequest(0);
};
export default wrappedFetch;