diff --git a/lib/config.ts b/lib/config.ts index 2bbd417bf..09d9c193c 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -4,7 +4,7 @@ import got from 'got'; let envs = process.env; -type Config = { +export type Config = { disallowRobot: boolean; enableCluster?: string; isPackage: boolean; @@ -41,7 +41,6 @@ type Config = { url_regex: string; }; proxyStrategy: string; - reverseProxyUrl?: string; pacUri?: string; pacScript?: string; authentication: { @@ -378,7 +377,6 @@ const calculateValue = () => { url_regex: envs.PROXY_URL_REGEX || '.*', }, proxyStrategy: envs.PROXY_STRATEGY || 'all', // all / on_retry - reverseProxyUrl: envs.REVERSE_PROXY_URL, pacUri: envs.PAC_URI, pacScript: envs.PAC_SCRIPT, // auth diff --git a/lib/index.ts b/lib/index.ts index a210498ec..633baddf2 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -1,3 +1,5 @@ +import '@/utils/request-wrapper' + import { serve } from '@hono/node-server' import { Handler, Hono } from 'hono' diff --git a/lib/utils/pac-proxy.js b/lib/utils/pac-proxy.ts similarity index 81% rename from lib/utils/pac-proxy.js rename to lib/utils/pac-proxy.ts index f592137e7..9585c3ec7 100644 --- a/lib/utils/pac-proxy.js +++ b/lib/utils/pac-proxy.ts @@ -1,9 +1,9 @@ -const config = require('@/config').value; -const logger = require('./logger'); +import { type Config } from '@/config'; +import logger from '@/utils/logger'; const possibleProtocol = ['http', 'https', 'ftp', 'file', 'data']; -const pacProxy = (pacUri, pacScript, proxyObj) => { +const pacProxy = (pacUri: Config['pacUri'], pacScript: Config['pacScript'], proxyObj: Config['proxy']) => { let pacUrlHandler = null; // Validate PAC_URI / PAC_SCRIPT @@ -17,26 +17,26 @@ const pacProxy = (pacUri, pacScript, proxyObj) => { if (pacUri && typeof pacUri === 'string') { try { pacUrlHandler = new URL(pacUri); - } catch (error) { - pacUri = null; + } catch (error: any) { + pacUri = undefined; pacUrlHandler = null; logger.error(`Parse PAC_URI error: ${error.stack}`); } } else { - pacUri = null; + pacUri = undefined; } // Check if PAC_URI has the right protocol - if (pacUri && !possibleProtocol.includes(pacUrlHandler?.protocol?.replace(':', ''))) { + if (pacUri && (!pacUrlHandler?.protocol || !possibleProtocol.includes(pacUrlHandler.protocol.replace(':', '')))) { logger.error(`Unsupported PAC protocol: ${pacUrlHandler?.protocol?.replace(':', '')}, expect one of ${possibleProtocol.join(', ')}`); - pacUri = null; + pacUri = undefined; pacUrlHandler = null; } // Validate proxyObj if (pacUrlHandler) { proxyObj.host = pacUrlHandler.hostname; - proxyObj.port = Number.parseInt(pacUrlHandler.port) || undefined; + proxyObj.port = pacUrlHandler.port; proxyObj.protocol = pacUrlHandler.protocol.replace(':', ''); } else { proxyObj.protocol = proxyObj.host = proxyObj.port = proxyObj.auth = undefined; @@ -69,7 +69,4 @@ const pacProxy = (pacUri, pacScript, proxyObj) => { }; }; -module.exports = { - pacProxy, - ...pacProxy(config.pacUri, config.pacScript, config.proxy), -}; +export default pacProxy diff --git a/lib/utils/request-wrapper.js b/lib/utils/request-wrapper.js deleted file mode 100644 index b7b6e2f80..000000000 --- a/lib/utils/request-wrapper.js +++ /dev/null @@ -1,131 +0,0 @@ -const config = require('@/config').value; -const proxyIsPAC = config.pacUri || config.pacScript; -const { proxyUri, proxyObj, proxyUrlHandler } = proxyIsPAC ? require('./pac-proxy') : require('./unify-proxy'); -const logger = require('./logger'); -const http = require('http'); -const https = require('https'); - -let agent = null; -if (proxyIsPAC) { - const { PacProxyAgent } = require('pac-proxy-agent'); - agent = new PacProxyAgent(`pac+${proxyUri}`); -} else if (proxyUri) { - if (proxyUri.startsWith('http')) { - const { HttpsProxyAgent } = require('https-proxy-agent'); - agent = new HttpsProxyAgent(proxyUri); - } else if (proxyUri.startsWith('socks')) { - const { SocksProxyAgent } = require('socks-proxy-agent'); - agent = new SocksProxyAgent(proxyUri); - } -} - -let proxyWrapper = () => false; -if (agent) { - const proxyRegex = new RegExp(proxyObj.url_regex); - const protocolMatch = (protocolLike) => protocolLike && protocolLike.toLowerCase().startsWith('http'); - - proxyWrapper = (url, options) => { - let urlHandler; - try { - urlHandler = new URL(url); - } catch { - // ignore - } - if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler.host)) { - options.agent = agent; - if (proxyObj.auth) { - options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`; - } - return true; - } - return false; - }; -} else if (config.reverseProxyUrl) { - proxyWrapper = (url, options) => { - const urlIn = options.url?.toString() || url; - const proxyRegex = new RegExp(proxyObj.url_regex); - if ( - proxyRegex.test(urlIn) && - !urlIn.startsWith(config.reverseProxyUrl) && - !options.cookieJar && - !options.headers['user-agent']?.startsWith('PixivIOSApp') && - !options.headers['x-goog-api-client'] && - options.headers.accept !== 'application/dns-json' - ) { - options.url = new URL(`${config.reverseProxyUrl}${encodeURIComponent(urlIn)}`); - - return true; - } - return false; - }; -} - -const requestWrapper = ( - url, - options = { - headers: {}, - } -) => { - options.headers = options.headers || {}; - const headersLowerCaseKeys = new Set(Object.keys(options.headers).map((key) => key.toLowerCase())); - - let prxied = false; - if (config.proxyStrategy === 'all') { - prxied = proxyWrapper(url, options); - } else if (config.proxyStrategy === 'on_retry' && options.retryCount) { - prxied = proxyWrapper(url, options); - } - if (prxied) { - logger.http(`Proxy for ${url}`); - } else { - logger.http(`Requesting ${url}`); - } - - // ua - if (!headersLowerCaseKeys.has('user-agent')) { - options.headers['user-agent'] = config.ua; - } - - // Accept - if (!headersLowerCaseKeys.has('Accept')) { - options.headers.Accept = '*/*'; - } - - let urlHandler; - try { - urlHandler = new URL(options.url || url); - } catch { - // ignore - } - - if ( - urlHandler && // referer - !headersLowerCaseKeys.has('referer') - ) { - options.headers.referer = urlHandler.origin; - } -}; - -const httpWrap = (func) => { - const origin = func; - return function (url, request) { - const args = Array.prototype.slice.call(arguments); - if (typeof url === 'object') { - if (url instanceof URL) { - requestWrapper(url.toString(), request); - } else { - const req = url; - requestWrapper(req.url || req.href || `${req.protocol}//${req.hostname || req.host}${req.path}`, req); - } - } else { - requestWrapper(url, request); - } - args[0] = request?.url || url; - return origin.apply(this, args); - }; -}; - -http.get = httpWrap(http.get); -https.get = httpWrap(https.get); -http.request = httpWrap(http.request); -https.request = httpWrap(https.request); diff --git a/lib/utils/request-wrapper.ts b/lib/utils/request-wrapper.ts new file mode 100644 index 000000000..1c8306c2a --- /dev/null +++ b/lib/utils/request-wrapper.ts @@ -0,0 +1,137 @@ +import { config } from '@/config'; +import logger from '@/utils/logger'; +import http, { type RequestOptions } from 'node:http'; +import https from 'node:https'; + +const proxyIsPAC = config.pacUri || config.pacScript; + +import pacProxy from './pac-proxy'; +import unifyProxy from './unify-proxy'; + +let proxyUri: string | undefined; +let proxyObj: Record | undefined; +let proxyUrlHandler: URL | null = null; +if (proxyIsPAC) { + const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy) + proxyUri = proxy.proxyUri; + proxyObj = proxy.proxyObj; + proxyUrlHandler = proxy.proxyUrlHandler; +} else { + const proxy = unifyProxy(config.proxyUri, config.proxy) + proxyUri = proxy.proxyUri; + proxyObj = proxy.proxyObj; + proxyUrlHandler = proxy.proxyUrlHandler; +} + +let agent = null; +if (proxyIsPAC) { + const { PacProxyAgent } = require('pac-proxy-agent'); + agent = new PacProxyAgent(`pac+${proxyUri}`); +} else if (proxyUri) { + if (proxyUri.startsWith('http')) { + const { HttpsProxyAgent } = require('https-proxy-agent'); + agent = new HttpsProxyAgent(proxyUri); + } else if (proxyUri.startsWith('socks')) { + const { SocksProxyAgent } = require('socks-proxy-agent'); + agent = new SocksProxyAgent(proxyUri); + } +} + +let proxyWrapper: (url: string, options: RequestOptions & { + headers: Record; +}) => boolean = () => false; + +if (agent) { + const proxyRegex = new RegExp(proxyObj.url_regex); + const protocolMatch = (protocolLike?: string | null) => protocolLike?.toLowerCase().startsWith('http'); + + proxyWrapper = (url, options) => { + let urlHandler; + try { + urlHandler = new URL(url); + } catch { + // ignore + } + if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler?.host)) { + options.agent = agent; + if (proxyObj.auth) { + options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`; + } + return true; + } + return false; + }; +} + +const requestWrapper = ( + url: string, + options: http.RequestOptions = {}, +) => { + options.headers = options.headers || {}; + + const optionsWithHeaders = options as http.RequestOptions & { + headers: Record; + }; + const headersLowerCaseKeys = new Set(Object.keys(optionsWithHeaders.headers).map((key) => key.toLowerCase())); + + let prxied = false; + if (config.proxyStrategy === 'all') { + prxied = proxyWrapper(url, optionsWithHeaders); + } else if (config.proxyStrategy === 'on_retry' && (optionsWithHeaders as any).retryCount) { + prxied = proxyWrapper(url, optionsWithHeaders); + } + if (prxied) { + logger.debug(`Proxy for ${url}`); + } else { + logger.debug(`Requesting ${url}`); + } + + // ua + if (!headersLowerCaseKeys.has('user-agent')) { + options.headers['user-agent'] = config.ua; + } + + // Accept + if (!headersLowerCaseKeys.has('Accept')) { + options.headers.Accept = '*/*'; + } + + let urlHandler; + try { + urlHandler = new URL(url); + } catch { + // ignore + } + + if ( + urlHandler && // referer + !headersLowerCaseKeys.has('referer') + ) { + options.headers.referer = urlHandler.origin; + } +}; + +const httpWrap = (func: typeof http.request) => { + const origin = func; + const warpped: typeof http.request = function (...args) { + let url: string; + let options: http.RequestOptions; + if (args[0] instanceof URL || typeof args[0] === 'string') { + url = args[0].toString(); + options = args[1] as http.RequestOptions; + } else { + options = args[0] as http.RequestOptions; + url = `${options.protocol}//${options.hostname || options.host}${options.path}` + } + requestWrapper(url, options); + + // @ts-ignore + return origin.apply(this, args); + }; + return warpped; +}; + +http.get = httpWrap(http.get); +https.get = httpWrap(https.get); +http.request = httpWrap(http.request); +https.request = httpWrap(https.request); diff --git a/lib/utils/unify-proxy.js b/lib/utils/unify-proxy.ts similarity index 92% rename from lib/utils/unify-proxy.js rename to lib/utils/unify-proxy.ts index b92421dad..2d1c05933 100644 --- a/lib/utils/unify-proxy.js +++ b/lib/utils/unify-proxy.ts @@ -1,15 +1,15 @@ -const config = require('@/config').value; -const logger = require('./logger'); +import { type Config } from '@/config'; +import logger from '@/utils/logger'; const defaultProtocol = 'http'; const possibleProtocol = ['http', 'https', 'socks', 'socks4', 'socks4a', 'socks5', 'socks5h']; -const unifyProxy = (proxyUri, proxyObj) => { +const unifyProxy = (proxyUri: Config['proxyUri'], proxyObj: Config['proxy']) => { proxyObj = proxyObj || {}; const [oriProxyUri, oriProxyObj] = [proxyUri, proxyObj]; proxyObj = { ...proxyObj }; - let proxyUrlHandler; + let proxyUrlHandler: URL | null = null; // PROXY_URI if (proxyUri && typeof proxyUri === 'string') { @@ -19,7 +19,7 @@ const unifyProxy = (proxyUri, proxyObj) => { } try { proxyUrlHandler = new URL(proxyUri); - } catch (error) { + } catch (error: any) { logger.error(`Parse PROXY_URI error: ${error.stack}`); } } @@ -52,7 +52,7 @@ const unifyProxy = (proxyUri, proxyObj) => { } else { logger.warn('PROXY_PORT is not set, leaving proxy agent to determine'); } - } catch (error) { + } catch (error: any) { logger.error(`Parse PROXY_HOST error: ${error.stack}`); } } else { @@ -90,7 +90,7 @@ const unifyProxy = (proxyUri, proxyObj) => { } proxyObj.protocol = protocol; proxyObj.host = proxyUrlHandler.hostname; - proxyObj.port = Number.parseInt(proxyUrlHandler.port) || undefined; + proxyObj.port = proxyUrlHandler.port; // trailing slash will cause puppeteer to throw net::ERR_NO_SUPPORTED_PROXIES, trim it proxyUri = proxyUrlHandler.href.endsWith('/') ? proxyUrlHandler.href.slice(0, -1) : proxyUrlHandler.href; isProxyValid = true; @@ -103,14 +103,11 @@ const unifyProxy = (proxyUri, proxyObj) => { logger.error('Proxy is disabled due to misconfiguration'); } proxyObj.protocol = proxyObj.host = proxyObj.port = proxyObj.auth = undefined; - proxyUri = null; + proxyUri = undefined; proxyUrlHandler = null; } return { proxyUri, proxyObj, proxyUrlHandler }; }; -module.exports = { - unifyProxy, - ...unifyProxy(config.proxyUri, config.proxy), -}; +export default unifyProxy