diff --git a/lib/config.ts b/lib/config.ts index 2b2796921..2834a3d88 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -37,6 +37,7 @@ export type Config = { }; // proxy proxyUri?: string; + proxyUris?: string[]; proxy: { protocol?: string; host?: string; @@ -44,6 +45,8 @@ export type Config = { auth?: string; url_regex: string; strategy: 'on_retry' | 'all'; + failoverTimeout?: number; + healthCheckInterval?: number; }; pacUri?: string; pacScript?: string; @@ -491,6 +494,11 @@ const calculateValue = () => { }, // proxy proxyUri: envs.PROXY_URI, + proxyUris: envs.PROXY_URIS + ? envs.PROXY_URIS.split(',') + .map((uri) => uri.trim()) + .filter(Boolean) + : undefined, proxy: { protocol: envs.PROXY_PROTOCOL, host: envs.PROXY_HOST, @@ -498,6 +506,8 @@ const calculateValue = () => { auth: envs.PROXY_AUTH, url_regex: envs.PROXY_URL_REGEX || '.*', strategy: envs.PROXY_STRATEGY || 'all', // all / on_retry + failoverTimeout: toInt(envs.PROXY_FAILOVER_TIMEOUT, 5000), + healthCheckInterval: toInt(envs.PROXY_HEALTH_CHECK_INTERVAL, 60000), }, pacUri: envs.PAC_URI, pacScript: envs.PAC_SCRIPT, diff --git a/lib/utils/proxy/index.ts b/lib/utils/proxy/index.ts index aa0c5c9a7..4eb86afd5 100644 --- a/lib/utils/proxy/index.ts +++ b/lib/utils/proxy/index.ts @@ -3,20 +3,72 @@ import { PacProxyAgent } from 'pac-proxy-agent'; import { HttpsProxyAgent } from 'https-proxy-agent'; import { SocksProxyAgent } from 'socks-proxy-agent'; import { ProxyAgent } from 'undici'; +import logger from '@/utils/logger'; const proxyIsPAC = config.pacUri || config.pacScript; import pacProxy from './pac-proxy'; import unifyProxy from './unify-proxy'; +import createMultiProxy, { type MultiProxyResult, type ProxyState } from './multi-proxy'; + +interface ProxyExport { + agent: PacProxyAgent | HttpsProxyAgent | SocksProxyAgent | null; + dispatcher: ProxyAgent | null; + proxyUri?: string; + proxyObj: Record; + proxyUrlHandler?: URL | null; + multiProxy?: MultiProxyResult; + getCurrentProxy: () => ProxyState | null; + markProxyFailed: (proxyUri: string) => void; + getAgentForProxy: (proxyState: ProxyState) => any; + getDispatcherForProxy: (proxyState: ProxyState) => ProxyAgent | null; +} let proxyUri: string | undefined; -let proxyObj: Record | undefined; +let proxyObj: Record = {}; let proxyUrlHandler: URL | null = null; +let multiProxy: MultiProxyResult | undefined; + +const createAgentForProxy = (uri: string, proxyObj: Record): any => { + if (uri.startsWith('http')) { + return new HttpsProxyAgent(uri, { + headers: { + 'proxy-authorization': proxyObj?.auth ? `Basic ${proxyObj.auth}` : undefined, + }, + }); + } else if (uri.startsWith('socks')) { + return new SocksProxyAgent(uri); + } + return null; +}; + +const createDispatcherForProxy = (uri: string, proxyObj: Record): ProxyAgent | null => { + if (uri.startsWith('http')) { + return new ProxyAgent({ + uri, + token: proxyObj?.auth ? `Basic ${proxyObj.auth}` : undefined, + requestTls: { + rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0', + }, + }); + } + return null; +}; + if (proxyIsPAC) { const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy); proxyUri = proxy.proxyUri; proxyObj = proxy.proxyObj; proxyUrlHandler = proxy.proxyUrlHandler; +} else if (config.proxyUris && config.proxyUris.length > 0) { + multiProxy = createMultiProxy(config.proxyUris, config.proxy); + proxyObj = multiProxy.proxyObj; + const currentProxy = multiProxy.getNextProxy(); + if (currentProxy) { + proxyUri = currentProxy.uri; + proxyUrlHandler = currentProxy.urlHandler; + } + logger.info(`Multi-proxy initialized with ${config.proxyUris.length} proxies`); } else { const proxy = unifyProxy(config.proxyUri, config.proxy); proxyUri = proxy.proxyUri; @@ -26,31 +78,63 @@ if (proxyIsPAC) { let agent: PacProxyAgent | HttpsProxyAgent | SocksProxyAgent | null = null; let dispatcher: ProxyAgent | null = null; -if (proxyIsPAC) { + +if (proxyIsPAC && proxyUri) { agent = new PacProxyAgent(`pac+${proxyUri}`); } else if (proxyUri) { - if (proxyUri.startsWith('http')) { - agent = new HttpsProxyAgent(proxyUri, { - headers: { - 'proxy-authorization': config.proxy?.auth ? `Basic ${config.proxy?.auth}` : undefined, - }, - }); - dispatcher = new ProxyAgent({ - uri: proxyUri, - token: config.proxy?.auth ? `Basic ${config.proxy?.auth}` : undefined, - requestTls: { - rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0', - }, - }); - } else if (proxyUri.startsWith('socks')) { - agent = new SocksProxyAgent(proxyUri); - } + agent = createAgentForProxy(proxyUri, proxyObj); + dispatcher = createDispatcherForProxy(proxyUri, proxyObj); } -export default { +const getCurrentProxy = (): ProxyState | null => { + if (multiProxy) { + return multiProxy.getNextProxy(); + } + if (proxyUri) { + return { + uri: proxyUri, + isActive: true, + failureCount: 0, + urlHandler: proxyUrlHandler, + }; + } + return null; +}; + +const markProxyFailed = (failedProxyUri: string) => { + if (multiProxy) { + multiProxy.markProxyFailed(failedProxyUri); + const nextProxy = multiProxy.getNextProxy(); + if (nextProxy) { + proxyUri = nextProxy.uri; + proxyUrlHandler = nextProxy.urlHandler || null; + agent = createAgentForProxy(nextProxy.uri, proxyObj); + dispatcher = createDispatcherForProxy(nextProxy.uri, proxyObj); + logger.info(`Switched to proxy: ${nextProxy.uri}`); + } else { + logger.warn('No available proxies remaining'); + agent = null; + dispatcher = null; + proxyUri = undefined; + } + } +}; + +const getAgentForProxy = (proxyState: ProxyState) => createAgentForProxy(proxyState.uri, proxyObj); + +const getDispatcherForProxy = (proxyState: ProxyState) => createDispatcherForProxy(proxyState.uri, proxyObj); + +const proxyExport: ProxyExport = { agent, dispatcher, proxyUri, proxyObj, proxyUrlHandler, + multiProxy, + getCurrentProxy, + markProxyFailed, + getAgentForProxy, + getDispatcherForProxy, }; + +export default proxyExport; diff --git a/lib/utils/proxy/multi-proxy.ts b/lib/utils/proxy/multi-proxy.ts new file mode 100644 index 000000000..38209f316 --- /dev/null +++ b/lib/utils/proxy/multi-proxy.ts @@ -0,0 +1,139 @@ +import { type Config } from '@/config'; +import logger from '@/utils/logger'; +import unifyProxy from './unify-proxy'; + +export interface ProxyState { + uri: string; + isActive: boolean; + failureCount: number; + lastFailureTime?: number; + agent?: any; + dispatcher?: any; + urlHandler?: URL | null; +} + +export interface MultiProxyResult { + currentProxy?: ProxyState | null; + allProxies: ProxyState[]; + proxyObj: Config['proxy']; + getNextProxy: () => ProxyState | null; + markProxyFailed: (proxyUri: string) => void; + resetProxy: (proxyUri: string) => void; +} + +const createMultiProxy = (proxyUris: string[], proxyObj: Config['proxy']): MultiProxyResult => { + const proxies: ProxyState[] = []; + let currentProxyIndex = 0; + + for (const uri of proxyUris) { + const unifiedProxy = unifyProxy(uri, proxyObj); + if (unifiedProxy.proxyUri) { + proxies.push({ + uri: unifiedProxy.proxyUri, + isActive: true, + failureCount: 0, + urlHandler: unifiedProxy.proxyUrlHandler, + }); + } + } + + if (proxies.length === 0) { + logger.warn('No valid proxies found in the provided list'); + return { + allProxies: [], + proxyObj: proxyObj || {}, + getNextProxy: () => null, + markProxyFailed: () => {}, + resetProxy: () => {}, + }; + } + + const healthCheckInterval = proxyObj?.healthCheckInterval || 60000; + const maxFailures = 3; + + const healthCheck = () => { + const now = Date.now(); + for (const proxy of proxies) { + if (!proxy.isActive && proxy.lastFailureTime && now - proxy.lastFailureTime > healthCheckInterval) { + proxy.isActive = true; + proxy.failureCount = 0; + delete proxy.lastFailureTime; + logger.info(`Proxy ${proxy.uri} marked as active again after health check`); + } + } + }; + + setInterval(healthCheck, healthCheckInterval); + + const getNextProxy = (): ProxyState | null => { + const activeProxies = proxies.filter((p) => p.isActive); + if (activeProxies.length === 0) { + logger.warn('No active proxies available'); + return null; + } + + let nextProxy = activeProxies[currentProxyIndex % activeProxies.length]; + let attempts = 0; + + while (!nextProxy.isActive && attempts < activeProxies.length) { + currentProxyIndex = (currentProxyIndex + 1) % activeProxies.length; + nextProxy = activeProxies[currentProxyIndex]; + attempts++; + } + + if (!nextProxy.isActive) { + return null; + } + + return nextProxy; + }; + + const markProxyFailed = (proxyUri: string) => { + const proxy = proxies.find((p) => p.uri === proxyUri); + if (proxy) { + proxy.failureCount++; + proxy.lastFailureTime = Date.now(); + if (proxy.failureCount >= maxFailures) { + proxy.isActive = false; + logger.warn(`Proxy ${proxyUri} marked as inactive after ${maxFailures} failures`); + } else { + logger.warn(`Proxy ${proxyUri} failed (${proxy.failureCount}/${maxFailures})`); + } + + const activeProxies = proxies.filter((p) => p.isActive); + if (activeProxies.length > 0) { + currentProxyIndex = (currentProxyIndex + 1) % activeProxies.length; + const nextProxy = getNextProxy(); + if (nextProxy) { + logger.info(`Switching to proxy: ${nextProxy.uri}`); + } + } + } + }; + + const resetProxy = (proxyUri: string) => { + const proxy = proxies.find((p) => p.uri === proxyUri); + if (proxy) { + proxy.isActive = true; + proxy.failureCount = 0; + delete proxy.lastFailureTime; + logger.info(`Proxy ${proxyUri} manually reset`); + } + }; + + const currentProxy = getNextProxy(); + if (currentProxy) { + logger.info(`Initial proxy selected: ${currentProxy.uri}`); + } + + return { + currentProxy, + allProxies: proxies, + proxyObj: proxyObj || {}, + getNextProxy, + markProxyFailed, + resetProxy, + }; +}; + +export default createMultiProxy; diff --git a/lib/utils/proxy/unify-proxy.ts b/lib/utils/proxy/unify-proxy.ts index aa7c33ff5..335aa376f 100644 --- a/lib/utils/proxy/unify-proxy.ts +++ b/lib/utils/proxy/unify-proxy.ts @@ -4,7 +4,7 @@ import logger from '@/utils/logger'; const defaultProtocol = 'http'; const possibleProtocol = ['http', 'https', 'socks', 'socks4', 'socks4a', 'socks5', 'socks5h']; -const unifyProxy = (proxyUri: Config['proxyUri'], proxyObj: Config['proxy']) => { +const unifyProxy = (proxyUri: Config['proxyUri'] | string, proxyObj: Config['proxy']) => { proxyObj = proxyObj || {}; const [oriProxyUri, oriProxyObj] = [proxyUri, proxyObj]; proxyObj = { ...proxyObj }; @@ -110,4 +110,6 @@ const unifyProxy = (proxyUri: Config['proxyUri'], proxyObj: Config['proxy']) => return { proxyUri, proxyObj, proxyUrlHandler }; }; +export const unifyProxies = (proxyUris: string[], proxyObj: Config['proxy']) => proxyUris.map((uri) => unifyProxy(uri, proxyObj)).filter((result) => result.proxyUri); + export default unifyProxy; diff --git a/lib/utils/puppeteer.ts b/lib/utils/puppeteer.ts index fd74e5f9c..0d6ef691c 100644 --- a/lib/utils/puppeteer.ts +++ b/lib/utils/puppeteer.ts @@ -30,17 +30,18 @@ const outPuppeteer = async ( insidePuppeteer.use(StealthPlugin()); } - if (proxy.proxyUri && proxy.proxyObj.url_regex === '.*') { - if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) { + const currentProxy = proxy.getCurrentProxy(); + if (currentProxy && proxy.proxyObj.url_regex === '.*') { + if (currentProxy.urlHandler?.username || currentProxy.urlHandler?.password) { // only proxies with authentication need to be anonymized - if (proxy.proxyUrlHandler.protocol === 'http:') { - options.args.push(`--proxy-server=${await anonymizeProxy(proxy.proxyUri)}`); + if (currentProxy.urlHandler.protocol === 'http:') { + options.args.push(`--proxy-server=${await anonymizeProxy(currentProxy.uri)}`); } else { logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy'); } } else { // Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes - options.args.push(`--proxy-server=${proxy.proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`); + options.args.push(`--proxy-server=${currentProxy.uri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`); } } const browser = await (config.puppeteerWSEndpoint @@ -101,11 +102,14 @@ export const getPuppeteerPage = async ( } let hasProxy = false; - if (proxy.proxyUri && allowProxy) { - if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) { + let currentProxyState: any = null; + const currentProxy = proxy.getCurrentProxy(); + if (currentProxy && allowProxy) { + currentProxyState = currentProxy; + if (currentProxy.urlHandler?.username || currentProxy.urlHandler?.password) { // only proxies with authentication need to be anonymized - if (proxy.proxyUrlHandler.protocol === 'http:') { - const urlObj = new URL(proxy.proxyUri); + if (currentProxy.urlHandler.protocol === 'http:') { + const urlObj = new URL(currentProxy.uri); urlObj.username = ''; urlObj.password = ''; options.args.push(`--proxy-server=${urlObj.toString().replace(/\/$/, '')}`); @@ -115,7 +119,7 @@ export const getPuppeteerPage = async ( } } else { // Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes - options.args.push(`--proxy-server=${proxy.proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`); + options.args.push(`--proxy-server=${currentProxy.uri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`); hasProxy = true; } } @@ -145,14 +149,14 @@ export const getPuppeteerPage = async ( const page = await browser.newPage(); - if (hasProxy) { - logger.debug(`Proxying request in puppeteer: ${url}`); + if (hasProxy && currentProxyState) { + logger.debug(`Proxying request in puppeteer via ${currentProxyState.uri}: ${url}`); } - if (hasProxy && (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password)) { + if (hasProxy && currentProxyState && (currentProxyState.urlHandler?.username || currentProxyState.urlHandler?.password)) { await page.authenticate({ - username: proxy.proxyUrlHandler?.username, - password: proxy.proxyUrlHandler?.password, + username: currentProxyState.urlHandler?.username, + password: currentProxyState.urlHandler?.password, }); } @@ -161,7 +165,16 @@ export const getPuppeteerPage = async ( } if (!instanceOptions.noGoto) { - await page.goto(url, instanceOptions.gotoConfig || { waitUntil: 'domcontentloaded' }); + try { + await page.goto(url, instanceOptions.gotoConfig || { waitUntil: 'domcontentloaded' }); + } catch (error) { + if (hasProxy && currentProxyState && proxy.multiProxy) { + logger.warn(`Puppeteer navigation failed with proxy ${currentProxyState.uri}, marking as failed: ${error}`); + proxy.markProxyFailed(currentProxyState.uri); + throw error; + } + throw error; + } } return { diff --git a/lib/utils/request-rewriter/fetch.ts b/lib/utils/request-rewriter/fetch.ts index a04d6a060..f9f9f13e8 100644 --- a/lib/utils/request-rewriter/fetch.ts +++ b/lib/utils/request-rewriter/fetch.ts @@ -60,7 +60,7 @@ const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: Requ config.enableRemoteDebugging && useCustomHeader(request.headers); // proxy - if (!init?.dispatcher && proxy.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) { + if (!init?.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) { const proxyRegex = new RegExp(proxy.proxyObj.url_regex); let urlHandler; try { @@ -70,13 +70,51 @@ const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: Requ } if (proxyRegex.test(request.url) && request.url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) { - options.dispatcher = proxy.dispatcher; - logger.debug(`Proxying request: ${request.url}`); + const currentProxy = proxy.getCurrentProxy(); + if (currentProxy) { + const dispatcher = proxy.getDispatcherForProxy(currentProxy); + if (dispatcher) { + options.dispatcher = dispatcher; + logger.debug(`Proxying request via ${currentProxy.uri}: ${request.url}`); + } + } } } await limiterQueue.removeTokens(1); - return undici.fetch(request, options); + + const maxRetries = proxy.multiProxy?.allProxies.length || 1; + + const attemptRequest = async (attempt: number): Promise => { + try { + return await undici.fetch(request, options); + } catch (error) { + if (options.dispatcher && proxy.multiProxy && attempt < maxRetries - 1) { + const currentProxy = proxy.getCurrentProxy(); + if (currentProxy) { + logger.warn(`Request failed with proxy ${currentProxy.uri}, trying next proxy: ${error}`); + proxy.markProxyFailed(currentProxy.uri); + + const nextProxy = proxy.getCurrentProxy(); + if (nextProxy && nextProxy.uri !== currentProxy.uri) { + const nextDispatcher = proxy.getDispatcherForProxy(nextProxy); + if (nextDispatcher) { + options.dispatcher = nextDispatcher; + } + logger.debug(`Retrying request with proxy ${nextProxy.uri}: ${request.url}`); + return attemptRequest(attempt + 1); + } else { + logger.warn('No more proxies available, trying without proxy'); + delete options.dispatcher; + return attemptRequest(attempt + 1); + } + } + } + throw error; + } + }; + + return attemptRequest(0); }; export default wrappedFetch;