diff --git a/lib/utils/proxy/index.ts b/lib/utils/proxy/index.ts new file mode 100644 index 000000000..b307f84cd --- /dev/null +++ b/lib/utils/proxy/index.ts @@ -0,0 +1,42 @@ +import { config } from '@/config'; + +const proxyIsPAC = config.pacUri || config.pacScript; + +import pacProxy from './pac-proxy'; +import unifyProxy from './unify-proxy'; + +let proxyUri: string | undefined; +let proxyObj: Record | undefined; +let proxyUrlHandler: URL | null = null; +if (proxyIsPAC) { + const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy); + proxyUri = proxy.proxyUri; + proxyObj = proxy.proxyObj; + proxyUrlHandler = proxy.proxyUrlHandler; +} else { + const proxy = unifyProxy(config.proxyUri, config.proxy); + proxyUri = proxy.proxyUri; + proxyObj = proxy.proxyObj; + proxyUrlHandler = proxy.proxyUrlHandler; +} + +let agent = null; +if (proxyIsPAC) { + const { PacProxyAgent } = require('pac-proxy-agent'); + agent = new PacProxyAgent(`pac+${proxyUri}`); +} else if (proxyUri) { + if (proxyUri.startsWith('http')) { + const { HttpsProxyAgent } = require('https-proxy-agent'); + agent = new HttpsProxyAgent(proxyUri); + } else if (proxyUri.startsWith('socks')) { + const { SocksProxyAgent } = require('socks-proxy-agent'); + agent = new SocksProxyAgent(proxyUri); + } +} + +export default { + agent, + proxyUri, + proxyObj, + proxyUrlHandler, +}; diff --git a/lib/utils/pac-proxy.ts b/lib/utils/proxy/pac-proxy.ts similarity index 100% rename from lib/utils/pac-proxy.ts rename to lib/utils/proxy/pac-proxy.ts diff --git a/lib/utils/unify-proxy.ts b/lib/utils/proxy/unify-proxy.ts similarity index 100% rename from lib/utils/unify-proxy.ts rename to lib/utils/proxy/unify-proxy.ts diff --git a/lib/utils/puppeteer.js b/lib/utils/puppeteer.ts similarity index 73% rename from lib/utils/puppeteer.js rename to lib/utils/puppeteer.ts index 03799d0c3..05b81c035 100644 --- a/lib/utils/puppeteer.js +++ b/lib/utils/puppeteer.ts @@ -1,12 +1,15 @@ -const config = require('@/config').value; -const { proxyUri, proxyUrlHandler } = require('./unify-proxy'); -let puppeteer = require('puppeteer'); -const proxyChain = require('proxy-chain'); -const logger = require('./logger'); +import { config } from '@/config'; +import puppeteer from 'puppeteer'; +import proxyChain from 'proxy-chain'; +import logger from './logger'; +import proxy from './proxy'; + +import { type PuppeteerExtra, addExtra } from 'puppeteer-extra'; +import StealthPlugin from 'puppeteer-extra-plugin-stealth'; const options = { args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`], - headless: 'new', + headless: 'new' as const, ignoreHTTPSErrors: true, }; @@ -15,10 +18,14 @@ const options = { * @param {boolean} extraOptions.stealth - Use puppeteer-extra-plugin-stealth * @returns Puppeteer browser */ -module.exports = async (extraOptions = {}) => { +const outPuppeteer = async ( + extraOptions: { + stealth?: boolean; + } = {} +) => { + let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer; if (extraOptions.stealth) { - const { addExtra } = require('puppeteer-extra'); - puppeteer = addExtra(puppeteer); + insidePuppeteer = addExtra(puppeteer); // workaround for vercel/nft #54, #283, #304 require('puppeteer-extra-plugin-stealth/evasions/chrome.app'); @@ -41,27 +48,27 @@ module.exports = async (extraOptions = {}) => { require('puppeteer-extra-plugin-user-preferences'); require('puppeteer-extra-plugin-user-data-dir'); - puppeteer.use(require('puppeteer-extra-plugin-stealth')()); + insidePuppeteer.use(StealthPlugin()); } - if (proxyUri) { - if (proxyUrlHandler.username || proxyUrlHandler.password) { + if (proxy.proxyUri) { + if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) { // only proxies with authentication need to be anonymized - if (proxyUrlHandler.protocol === 'http:') { - options.args.push(`--proxy-server=${await proxyChain.anonymizeProxy(proxyUri)}`); + if (proxy.proxyUrlHandler.protocol === 'http:') { + options.args.push(`--proxy-server=${await proxyChain.anonymizeProxy(proxy.proxyUri)}`); } else { logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy'); } } else { // Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes - options.args.push(`--proxy-server=${proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`); + options.args.push(`--proxy-server=${proxy.proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`); } } const browser = await (config.puppeteerWSEndpoint - ? puppeteer.connect({ + ? insidePuppeteer.connect({ browserWSEndpoint: config.puppeteerWSEndpoint, }) - : puppeteer.launch( + : insidePuppeteer.launch( config.chromiumExecutablePath ? { executablePath: config.chromiumExecutablePath, @@ -75,3 +82,5 @@ module.exports = async (extraOptions = {}) => { return browser; }; + +export default outPuppeteer; diff --git a/lib/utils/request-wrapper.ts b/lib/utils/request-wrapper.ts index 3380eb3e2..d80fbc7f7 100644 --- a/lib/utils/request-wrapper.ts +++ b/lib/utils/request-wrapper.ts @@ -2,40 +2,7 @@ import { config } from '@/config'; import logger from '@/utils/logger'; import http, { type RequestOptions } from 'node:http'; import https from 'node:https'; - -const proxyIsPAC = config.pacUri || config.pacScript; - -import pacProxy from './pac-proxy'; -import unifyProxy from './unify-proxy'; - -let proxyUri: string | undefined; -let proxyObj: Record | undefined; -let proxyUrlHandler: URL | null = null; -if (proxyIsPAC) { - const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy); - proxyUri = proxy.proxyUri; - proxyObj = proxy.proxyObj; - proxyUrlHandler = proxy.proxyUrlHandler; -} else { - const proxy = unifyProxy(config.proxyUri, config.proxy); - proxyUri = proxy.proxyUri; - proxyObj = proxy.proxyObj; - proxyUrlHandler = proxy.proxyUrlHandler; -} - -let agent = null; -if (proxyIsPAC) { - const { PacProxyAgent } = require('pac-proxy-agent'); - agent = new PacProxyAgent(`pac+${proxyUri}`); -} else if (proxyUri) { - if (proxyUri.startsWith('http')) { - const { HttpsProxyAgent } = require('https-proxy-agent'); - agent = new HttpsProxyAgent(proxyUri); - } else if (proxyUri.startsWith('socks')) { - const { SocksProxyAgent } = require('socks-proxy-agent'); - agent = new SocksProxyAgent(proxyUri); - } -} +import proxy from '@/utils/proxy'; let proxyWrapper: ( url: string, @@ -44,8 +11,8 @@ let proxyWrapper: ( } ) => boolean = () => false; -if (agent) { - const proxyRegex = new RegExp(proxyObj.url_regex); +if (proxy.agent) { + const proxyRegex = new RegExp(proxy.proxyObj.url_regex); const protocolMatch = (protocolLike?: string | null) => protocolLike?.toLowerCase().startsWith('http'); proxyWrapper = (url, options) => { @@ -55,10 +22,10 @@ if (agent) { } catch { // ignore } - if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler?.host)) { - options.agent = agent; - if (proxyObj.auth) { - options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`; + if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxy.proxyUrlHandler?.host)) { + options.agent = proxy.agent || false; + if (proxy.proxyObj.auth) { + options.headers['Proxy-Authorization'] = `Basic ${proxy.proxyObj.auth}`; } return true; } diff --git a/lib/utils/wait.js b/lib/utils/wait.ts similarity index 60% rename from lib/utils/wait.js rename to lib/utils/wait.ts index 8219ed98a..3b862b0f3 100644 --- a/lib/utils/wait.js +++ b/lib/utils/wait.ts @@ -1,5 +1,7 @@ -module.exports = function wait(ms) { +function wait(ms: number) { return new Promise((resolve) => { setTimeout(resolve, ms); }); -}; +} + +export default wait; diff --git a/test/utils/puppeteer.js b/test/utils/puppeteer.ts similarity index 70% rename from test/utils/puppeteer.js rename to test/utils/puppeteer.ts index 2d5734a62..0db8224e3 100644 --- a/test/utils/puppeteer.js +++ b/test/utils/puppeteer.ts @@ -1,7 +1,9 @@ -let puppeteer; -const wait = require('../../lib/utils/wait'); +import { describe, expect, it, jest, afterEach } from '@jest/globals'; +import wait from '../../lib/utils/wait'; +import puppeteer from '../../lib/utils/puppeteer'; +import { type Browser } from 'puppeteer'; -let browser = null; +let browser: Browser | null = null; afterEach(() => { if (browser) { @@ -22,7 +24,6 @@ afterEach(() => { describe('puppeteer', () => { it('puppeteer run', async () => { - puppeteer = require('../../lib/utils/puppeteer'); browser = await puppeteer(); const startTime = Date.now(); const page = await browser.newPage(); @@ -33,23 +34,22 @@ describe('puppeteer', () => { const html = await page.evaluate(() => document.body.innerHTML); expect(html.length).toBeGreaterThan(0); - expect(browser.process().exitCode).toBe(null); // browser is still running + expect(browser.process()?.exitCode).toBe(null); // browser is still running const sleepTime = 31 * 1000 - (Date.now() - startTime); // prevent long loading time from failing the test if (sleepTime > 0) { await wait(sleepTime); } - expect(browser.process().exitCode).toBe(0); // browser is closed + expect(browser.process()?.exitCode).toBe(0); // browser is closed browser = null; }, 45000); if (!process.env.GITHUB_ACTIONS) { it('puppeteer without stealth', async () => { - puppeteer = require('../../lib/utils/puppeteer'); browser = await puppeteer({ stealth: false }); const page = await browser.newPage(); await page.goto('https://bot.sannysoft.com', { waitUntil: 'networkidle0' }); // page rendering is not instant, wait for expected elements to appear - const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.failed`).then((hd) => hd.evaluate((e) => e.textContent)))); + const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.failed`).then((hd) => hd?.evaluate((e) => e.textContent)))); // the website return empty string from time to time for no reason // since we don't really care whether puppeteer without stealth passes the bot test, just let it go expect(['present (failed)', '']).toContain(webDriverTest); @@ -57,12 +57,11 @@ describe('puppeteer', () => { }, 15000); it('puppeteer with stealth', async () => { - puppeteer = require('../../lib/utils/puppeteer'); browser = await puppeteer({ stealth: true }); const page = await browser.newPage(); await page.goto('https://bot.sannysoft.com', { waitUntil: 'networkidle0' }); // page rendering is not instant, wait for expected elements to appear - const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.passed`).then((hd) => hd.evaluate((e) => e.textContent)))); + const [webDriverTest, chromeTest] = await Promise.all(['webdriver', 'chrome'].map((t) => page.waitForSelector(`td#${t}-result.result.passed`).then((hd) => hd?.evaluate((e) => e.textContent)))); // these are something we really care about expect(webDriverTest).toBe('missing (passed)'); expect(chromeTest).toBe('present (passed)'); @@ -72,29 +71,26 @@ describe('puppeteer', () => { it('puppeteer accept http proxy uri w/ auth', async () => { process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333'; - puppeteer = require('../../lib/utils/puppeteer'); browser = await puppeteer(); // trailing slash will cause net::ERR_NO_SUPPORTED_PROXIES, prohibit it - expect(browser.process().spawnargs.some((arg) => /^--proxy-server=http:\/\/.*[^/]$/.test(arg))).toBe(true); + expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=http:\/\/.*[^/]$/.test(arg))).toBe(true); }); it('puppeteer reject https proxy uri w/ auth', async () => { process.env.PROXY_URI = 'https://user:pass@rsshub.proxy:2333'; - puppeteer = require('../../lib/utils/puppeteer'); browser = await puppeteer(); - expect(browser.process().spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false); + expect(browser.process()?.spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false); }); it('puppeteer reject socks proxy uri w/ auth', async () => { process.env.PROXY_URI = 'socks5://user:pass@rsshub.proxy:2333'; - puppeteer = require('../../lib/utils/puppeteer'); browser = await puppeteer(); - expect(browser.process().spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false); + expect(browser.process()?.spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false); }); it('puppeteer accept http proxy', async () => { @@ -102,10 +98,9 @@ describe('puppeteer', () => { process.env.PROXY_HOST = 'rsshub.proxy'; process.env.PROXY_PORT = '2333'; - puppeteer = require('../../lib/utils/puppeteer'); browser = await puppeteer(); - expect(browser.process().spawnargs.some((arg) => /^--proxy-server=http:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true); + expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=http:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true); }, 10000); it('puppeteer accept https proxy', async () => { @@ -113,10 +108,9 @@ describe('puppeteer', () => { process.env.PROXY_HOST = 'rsshub.proxy'; process.env.PROXY_PORT = '2333'; - puppeteer = require('../../lib/utils/puppeteer'); browser = await puppeteer(); - expect(browser.process().spawnargs.some((arg) => /^--proxy-server=https:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true); + expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=https:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true); }, 10000); it('puppeteer accept socks4a proxy', async () => { @@ -124,10 +118,9 @@ describe('puppeteer', () => { process.env.PROXY_HOST = 'rsshub.proxy'; process.env.PROXY_PORT = '2333'; - puppeteer = require('../../lib/utils/puppeteer'); browser = await puppeteer(); - expect(browser.process().spawnargs.some((arg) => /^--proxy-server=socks4:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true); + expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=socks4:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true); }, 10000); it('puppeteer accept socks5h proxy', async () => { @@ -135,9 +128,8 @@ describe('puppeteer', () => { process.env.PROXY_HOST = 'rsshub.proxy'; process.env.PROXY_PORT = '2333'; - puppeteer = require('../../lib/utils/puppeteer'); browser = await puppeteer(); - expect(browser.process().spawnargs.some((arg) => /^--proxy-server=socks5:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true); + expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=socks5:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true); }, 10000); }); diff --git a/tsconfig.json b/tsconfig.json index b14fe83f3..561edef6f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,6 +7,7 @@ "paths": { "@/*": ["./lib/*"] }, - "esModuleInterop": true + "esModuleInterop": true, + "noImplicitAny": false, } }