From bf3649324bcc9abd07ad8d799152add54e1e797b Mon Sep 17 00:00:00 2001 From: DIYgod Date: Wed, 28 May 2025 21:12:02 +0800 Subject: [PATCH] fix: puppeteer proxy regex --- lib/routes/picnob/user.ts | 142 +++++++++++++++++------------------- lib/routes/picnob/utils.ts | 26 ++++--- lib/utils/puppeteer.test.ts | 119 ++++++++++++++++++++++++++++++ lib/utils/puppeteer.ts | 122 +++++++++++++++++++++++++++++-- 4 files changed, 313 insertions(+), 96 deletions(-) diff --git a/lib/routes/picnob/user.ts b/lib/routes/picnob/user.ts index 40bddab98..e46e42221 100644 --- a/lib/routes/picnob/user.ts +++ b/lib/routes/picnob/user.ts @@ -4,7 +4,6 @@ import ofetch from '@/utils/ofetch'; import { load } from 'cheerio'; import { parseRelativeDate } from '@/utils/parse-date'; import { puppeteerGet } from './utils'; -import puppeteer from '@/utils/puppeteer'; import sanitizeHtml from 'sanitize-html'; import cache from '@/utils/cache'; @@ -47,82 +46,73 @@ async function handler(ctx) { const type = ctx.req.param('type') ?? 'profile'; const profileUrl = `${baseUrl}/profile/${id}/${type === 'tagged' ? 'tagged/' : ''}`; - const browser = await puppeteer(); // TODO: can't bypass cloudflare 403 error without puppeteer - + let html; + let usePuppeteer = false; try { - let html; - let usePuppeteer = false; - try { - const data = await ofetch(profileUrl); - html = data; - } catch { - html = await puppeteerGet(profileUrl, browser); - usePuppeteer = true; - } - const $ = load(html); - - const list = $('.post_box') - .toArray() - .map((item) => { - const $item = $(item); - const sum = $item.find('.sum').text(); - const coverLink = $item.find('.cover_link').attr('href'); - const shortcode = coverLink?.split('/')?.[2]; - - return { - title: sanitizeHtml(sum.split('\n')[0], { allowedTags: [], allowedAttributes: {} }), - description: `
${sum.replaceAll('\n', '
')}`, - link: `${baseUrl}${coverLink}`, - guid: shortcode, - pubDate: parseRelativeDate($item.find('.time .txt').text()), - }; - }); - - const newDescription = await Promise.all( - list.map((item) => - cache.tryGet(`picnob:user:${id}:${item.guid}`, async () => { - try { - const html = usePuppeteer - ? await puppeteerGet(item.link, browser) - : await ofetch(item.link, { - headers: { - 'user-agent': 'PostmanRuntime/7.44.0', - }, - }); - const $ = load(html); - if ($('.video_img').length > 0) { - return `
${$('.sum_full').text()}`; - } else { - let description = ''; - for (const slide of $('.swiper-slide').toArray()) { - const $slide = $(slide); - description += `
`; - } - description += $('.sum_full').text(); - return description; - } - } catch { - return ''; - } - }) - ) - ); - - return { - title: `${$('h1.fullname').text()} (@${id}) ${type === 'tagged' ? 'tagged' : 'public'} posts - Picnob`, - description: $('.info .sum').text(), - link: profileUrl, - image: $('.ava .pic img').attr('src'), - item: list.map((item, index) => ({ - ...item, - description: newDescription[index] || item.description, - })), - }; - } catch (error) { - await browser.close(); - throw error; - } finally { - await browser.close(); + const data = await ofetch(profileUrl); + html = data; + } catch { + html = await puppeteerGet(profileUrl); + usePuppeteer = true; } + const $ = load(html); + + const list = $('.post_box') + .toArray() + .map((item) => { + const $item = $(item); + const sum = $item.find('.sum').text(); + const coverLink = $item.find('.cover_link').attr('href'); + const shortcode = coverLink?.split('/')?.[2]; + + return { + title: sanitizeHtml(sum.split('\n')[0], { allowedTags: [], allowedAttributes: {} }), + description: `
${sum.replaceAll('\n', '
')}`, + link: `${baseUrl}${coverLink}`, + guid: shortcode, + pubDate: parseRelativeDate($item.find('.time .txt').text()), + }; + }); + + const newDescription = await Promise.all( + list.map((item) => + cache.tryGet(`picnob:user:${id}:${item.guid}`, async () => { + try { + const html = usePuppeteer + ? await puppeteerGet(item.link) + : await ofetch(item.link, { + headers: { + 'user-agent': 'PostmanRuntime/7.44.0', + }, + }); + const $ = load(html); + if ($('.video_img').length > 0) { + return `
${$('.sum_full').text()}`; + } else { + let description = ''; + for (const slide of $('.swiper-slide').toArray()) { + const $slide = $(slide); + description += `
`; + } + description += $('.sum_full').text(); + return description; + } + } catch { + return ''; + } + }) + ) + ); + + return { + title: `${$('h1.fullname').text()} (@${id}) ${type === 'tagged' ? 'tagged' : 'public'} posts - Picnob`, + description: $('.info .sum').text(), + link: profileUrl, + image: $('.ava .pic img').attr('src'), + item: list.map((item, index) => ({ + ...item, + description: newDescription[index] || item.description, + })), + }; } diff --git a/lib/routes/picnob/utils.ts b/lib/routes/picnob/utils.ts index e9a6e56f0..94cda7c88 100644 --- a/lib/routes/picnob/utils.ts +++ b/lib/routes/picnob/utils.ts @@ -1,17 +1,19 @@ -const puppeteerGet = async (url, browser) => { +import { getPuppeteerPage } from '@/utils/puppeteer'; + +const puppeteerGet = async (url) => { let data; - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' ? request.continue() : request.abort(); + const { destory } = await getPuppeteerPage(url, { + onBeforeLoad: async (page) => { + await page.setRequestInterception(true); + page.on('request', (request) => { + request.resourceType() === 'document' ? request.continue() : request.abort(); + }); + page.on('response', async (response) => { + data = await (response.request().url().includes('/api/posts') ? response.json() : response.text()); + }); + }, }); - page.on('response', async (response) => { - data = await (response.request().url().includes('/api/posts') ? response.json() : response.text()); - }); - await page.goto(url, { - waitUntil: 'domcontentloaded', - }); - await page.close(); + await destory(); return data; }; diff --git a/lib/utils/puppeteer.test.ts b/lib/utils/puppeteer.test.ts index 1fd8f1d5a..e276cb7c7 100644 --- a/lib/utils/puppeteer.test.ts +++ b/lib/utils/puppeteer.test.ts @@ -142,3 +142,122 @@ describe('puppeteer', () => { expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=socks5:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true); }, 10000); }); + +describe('getPuppeteerPage', () => { + it('puppeteer run', async () => { + const { getPuppeteerPage } = await import('./puppeteer'); + const pup = await getPuppeteerPage('https://www.google.com'); + const page = pup.page; + browser = pup.browser; + const startTime = Date.now(); + + const html = await page.evaluate(() => document.body.innerHTML); + expect(html.length).toBeGreaterThan(0); + + expect(browser.process()?.exitCode).toBe(null); // browser is still running + const sleepTime = 31 * 1000 - (Date.now() - startTime); // prevent long loading time from failing the test + if (sleepTime > 0) { + await wait(sleepTime); + } + expect(browser.process()?.exitCode).toBe(0); // browser is closed + }, 45000); + + it('puppeteer accept http proxy uri w/ auth', async () => { + process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333'; + + const { getPuppeteerPage } = await import('./puppeteer'); + const pup = await getPuppeteerPage('https://www.google.com'); + browser = pup.browser; + + // trailing slash will cause net::ERR_NO_SUPPORTED_PROXIES, prohibit it + expect(browser.process()?.spawnargs.includes('--proxy-server=http://rsshub.proxy:2333')).toBe(true); + }); + + it('puppeteer respect proxy regex', async () => { + process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333'; + process.env.PROXY_URL_REGEX = 'not-exist'; + + const { getPuppeteerPage } = await import('./puppeteer'); + const pup = await getPuppeteerPage('https://www.google.com'); + browser = pup.browser; + + // trailing slash will cause net::ERR_NO_SUPPORTED_PROXIES, prohibit it + expect(browser.process()?.spawnargs.includes('--proxy-server=http://rsshub.proxy:2333')).toBe(false); + }); + + it('puppeteer reject https proxy uri w/ auth', async () => { + process.env.PROXY_URI = 'https://user:pass@rsshub.proxy:2333'; + + const { getPuppeteerPage } = await import('./puppeteer'); + const pup = await getPuppeteerPage('https://www.google.com'); + browser = pup.browser; + + expect(browser.process()?.spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false); + }); + + it('puppeteer reject socks proxy uri w/ auth', async () => { + process.env.PROXY_URI = 'socks5://user:pass@rsshub.proxy:2333'; + + const { getPuppeteerPage } = await import('./puppeteer'); + const pup = await getPuppeteerPage('https://www.google.com'); + browser = pup.browser; + + expect(browser.process()?.spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false); + }); + + it('puppeteer accept http proxy', async () => { + process.env.PROXY_PROTOCOL = 'http'; + process.env.PROXY_HOST = 'rsshub.proxy'; + process.env.PROXY_PORT = '2333'; + + const { getPuppeteerPage } = await import('./puppeteer'); + const pup = await getPuppeteerPage('https://www.google.com', { + noGoto: true, + }); + browser = pup.browser; + + expect(browser.process()?.spawnargs.includes('--proxy-server=http://rsshub.proxy:2333')).toBe(true); + }, 10000); + + it('puppeteer accept https proxy', async () => { + process.env.PROXY_PROTOCOL = 'https'; + process.env.PROXY_HOST = 'rsshub.proxy'; + process.env.PROXY_PORT = '2333'; + + const { getPuppeteerPage } = await import('./puppeteer'); + const pup = await getPuppeteerPage('https://www.google.com', { + noGoto: true, + }); + browser = pup.browser; + + expect(browser.process()?.spawnargs.includes('--proxy-server=https://rsshub.proxy:2333')).toBe(true); + }, 10000); + + it('puppeteer accept socks4a proxy', async () => { + process.env.PROXY_PROTOCOL = 'socks4a'; + process.env.PROXY_HOST = 'rsshub.proxy'; + process.env.PROXY_PORT = '2333'; + + const { getPuppeteerPage } = await import('./puppeteer'); + const pup = await getPuppeteerPage('https://www.google.com', { + noGoto: true, + }); + browser = pup.browser; + + expect(browser.process()?.spawnargs.includes('--proxy-server=socks4://rsshub.proxy:2333')).toBe(true); + }, 10000); + + it('puppeteer accept socks5h proxy', async () => { + process.env.PROXY_PROTOCOL = 'socks5h'; + process.env.PROXY_HOST = 'rsshub.proxy'; + process.env.PROXY_PORT = '2333'; + + const { getPuppeteerPage } = await import('./puppeteer'); + const pup = await getPuppeteerPage('https://www.google.com', { + noGoto: true, + }); + browser = pup.browser; + + expect(browser.process()?.spawnargs.includes('--proxy-server=socks5://rsshub.proxy:2333')).toBe(true); + }, 10000); +}); diff --git a/lib/utils/puppeteer.ts b/lib/utils/puppeteer.ts index 2d97ef96b..99aa591dc 100644 --- a/lib/utils/puppeteer.ts +++ b/lib/utils/puppeteer.ts @@ -1,5 +1,5 @@ import { config } from '@/config'; -import puppeteer from 'puppeteer'; +import puppeteer, { Browser, Page } from 'puppeteer'; import logger from './logger'; import proxy from './proxy'; import { anonymizeProxy } from 'proxy-chain'; @@ -7,13 +7,8 @@ import { anonymizeProxy } from 'proxy-chain'; import { type PuppeteerExtra, addExtra } from 'puppeteer-extra'; import StealthPlugin from 'puppeteer-extra-plugin-stealth'; -const options = { - args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`], - headless: true, - ignoreHTTPSErrors: true, -}; - /** + * @deprecated use getPage instead * @param {Object} extraOptions * @param {boolean} extraOptions.stealth - Use puppeteer-extra-plugin-stealth * @returns Puppeteer browser @@ -23,13 +18,19 @@ const outPuppeteer = async ( stealth?: boolean; } = {} ) => { + const options = { + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`], + headless: true, + ignoreHTTPSErrors: true, + }; + let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer; if (extraOptions.stealth) { insidePuppeteer = addExtra(puppeteer); insidePuppeteer.use(StealthPlugin()); } - if (proxy.proxyUri) { + if (proxy.proxyUri && proxy.proxyObj.url_regex === '.*') { if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) { // only proxies with authentication need to be anonymized if (proxy.proxyUrlHandler.protocol === 'http:') { @@ -62,3 +63,108 @@ const outPuppeteer = async ( }; export default outPuppeteer; + +/** + * @returns Puppeteer page + */ +export const getPuppeteerPage = async ( + url: string, + instanceOptions: { + onBeforeLoad?: (page: Page, browser?: Browser) => Promise | void; + gotoConfig?: { + waitUntil?: 'load' | 'domcontentloaded' | 'networkidle0'; + }; + noGoto?: boolean; + } = {} +) => { + const options = { + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`], + headless: true, + ignoreHTTPSErrors: true, + }; + + let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer; + insidePuppeteer = addExtra(puppeteer); + insidePuppeteer.use(StealthPlugin()); + + let allowProxy = false; + const proxyRegex = new RegExp(proxy.proxyObj.url_regex); + let urlHandler; + try { + urlHandler = new URL(url); + } catch { + // ignore + } + + if (proxyRegex.test(url) && url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) { + allowProxy = true; + } + + let hasProxy = false; + if (proxy.proxyUri && allowProxy) { + if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) { + // only proxies with authentication need to be anonymized + if (proxy.proxyUrlHandler.protocol === 'http:') { + const urlObj = new URL(proxy.proxyUri); + urlObj.username = ''; + urlObj.password = ''; + options.args.push(`--proxy-server=${urlObj.toString().replace(/\/$/, '')}`); + hasProxy = true; + } else { + logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy'); + } + } else { + // Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes + options.args.push(`--proxy-server=${proxy.proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`); + hasProxy = true; + } + } + let browser: Browser; + if (config.puppeteerWSEndpoint) { + const endpointURL = new URL(config.puppeteerWSEndpoint); + endpointURL.searchParams.set('launch', JSON.stringify(options)); + endpointURL.searchParams.set('stealth', 'true'); + const endpoint = endpointURL.toString(); + browser = await insidePuppeteer.connect({ + browserWSEndpoint: endpoint, + }); + } else { + browser = await insidePuppeteer.launch( + config.chromiumExecutablePath + ? { + executablePath: config.chromiumExecutablePath, + ...options, + } + : options + ); + } + + setTimeout(async () => { + await browser.close(); + }, 30000); + + const page = await browser.newPage(); + + if (hasProxy && (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password)) { + await page.authenticate({ + username: proxy.proxyUrlHandler?.username, + password: proxy.proxyUrlHandler?.password, + }); + } + + if (instanceOptions.onBeforeLoad) { + await instanceOptions.onBeforeLoad(page, browser); + } + + if (!instanceOptions.noGoto) { + await page.goto(url, instanceOptions.gotoConfig || { waitUntil: 'domcontentloaded' }); + } + + return { + page, + destory: async () => { + await browser.close(); + }, + browser, + }; +};