diff --git a/lib/routes/picnob/user.ts b/lib/routes/picnob/user.ts
index 40bddab98..e46e42221 100644
--- a/lib/routes/picnob/user.ts
+++ b/lib/routes/picnob/user.ts
@@ -4,7 +4,6 @@ import ofetch from '@/utils/ofetch';
import { load } from 'cheerio';
import { parseRelativeDate } from '@/utils/parse-date';
import { puppeteerGet } from './utils';
-import puppeteer from '@/utils/puppeteer';
import sanitizeHtml from 'sanitize-html';
import cache from '@/utils/cache';
@@ -47,82 +46,73 @@ async function handler(ctx) {
const type = ctx.req.param('type') ?? 'profile';
const profileUrl = `${baseUrl}/profile/${id}/${type === 'tagged' ? 'tagged/' : ''}`;
- const browser = await puppeteer();
// TODO: can't bypass cloudflare 403 error without puppeteer
-
+ let html;
+ let usePuppeteer = false;
try {
- let html;
- let usePuppeteer = false;
- try {
- const data = await ofetch(profileUrl);
- html = data;
- } catch {
- html = await puppeteerGet(profileUrl, browser);
- usePuppeteer = true;
- }
- const $ = load(html);
-
- const list = $('.post_box')
- .toArray()
- .map((item) => {
- const $item = $(item);
- const sum = $item.find('.sum').text();
- const coverLink = $item.find('.cover_link').attr('href');
- const shortcode = coverLink?.split('/')?.[2];
-
- return {
- title: sanitizeHtml(sum.split('\n')[0], { allowedTags: [], allowedAttributes: {} }),
- description: `.attr('data-src')})
${sum.replaceAll('\n', '
')}`,
- link: `${baseUrl}${coverLink}`,
- guid: shortcode,
- pubDate: parseRelativeDate($item.find('.time .txt').text()),
- };
- });
-
- const newDescription = await Promise.all(
- list.map((item) =>
- cache.tryGet(`picnob:user:${id}:${item.guid}`, async () => {
- try {
- const html = usePuppeteer
- ? await puppeteerGet(item.link, browser)
- : await ofetch(item.link, {
- headers: {
- 'user-agent': 'PostmanRuntime/7.44.0',
- },
- });
- const $ = load(html);
- if ($('.video_img').length > 0) {
- return `
${$('.sum_full').text()}`;
- } else {
- let description = '';
- for (const slide of $('.swiper-slide').toArray()) {
- const $slide = $(slide);
- description += `.attr('data-src')})
`;
- }
- description += $('.sum_full').text();
- return description;
- }
- } catch {
- return '';
- }
- })
- )
- );
-
- return {
- title: `${$('h1.fullname').text()} (@${id}) ${type === 'tagged' ? 'tagged' : 'public'} posts - Picnob`,
- description: $('.info .sum').text(),
- link: profileUrl,
- image: $('.ava .pic img').attr('src'),
- item: list.map((item, index) => ({
- ...item,
- description: newDescription[index] || item.description,
- })),
- };
- } catch (error) {
- await browser.close();
- throw error;
- } finally {
- await browser.close();
+ const data = await ofetch(profileUrl);
+ html = data;
+ } catch {
+ html = await puppeteerGet(profileUrl);
+ usePuppeteer = true;
}
+ const $ = load(html);
+
+ const list = $('.post_box')
+ .toArray()
+ .map((item) => {
+ const $item = $(item);
+ const sum = $item.find('.sum').text();
+ const coverLink = $item.find('.cover_link').attr('href');
+ const shortcode = coverLink?.split('/')?.[2];
+
+ return {
+ title: sanitizeHtml(sum.split('\n')[0], { allowedTags: [], allowedAttributes: {} }),
+ description: `.attr('data-src')})
${sum.replaceAll('\n', '
')}`,
+ link: `${baseUrl}${coverLink}`,
+ guid: shortcode,
+ pubDate: parseRelativeDate($item.find('.time .txt').text()),
+ };
+ });
+
+ const newDescription = await Promise.all(
+ list.map((item) =>
+ cache.tryGet(`picnob:user:${id}:${item.guid}`, async () => {
+ try {
+ const html = usePuppeteer
+ ? await puppeteerGet(item.link)
+ : await ofetch(item.link, {
+ headers: {
+ 'user-agent': 'PostmanRuntime/7.44.0',
+ },
+ });
+ const $ = load(html);
+ if ($('.video_img').length > 0) {
+ return `
${$('.sum_full').text()}`;
+ } else {
+ let description = '';
+ for (const slide of $('.swiper-slide').toArray()) {
+ const $slide = $(slide);
+ description += `.attr('data-src')})
`;
+ }
+ description += $('.sum_full').text();
+ return description;
+ }
+ } catch {
+ return '';
+ }
+ })
+ )
+ );
+
+ return {
+ title: `${$('h1.fullname').text()} (@${id}) ${type === 'tagged' ? 'tagged' : 'public'} posts - Picnob`,
+ description: $('.info .sum').text(),
+ link: profileUrl,
+ image: $('.ava .pic img').attr('src'),
+ item: list.map((item, index) => ({
+ ...item,
+ description: newDescription[index] || item.description,
+ })),
+ };
}
diff --git a/lib/routes/picnob/utils.ts b/lib/routes/picnob/utils.ts
index e9a6e56f0..94cda7c88 100644
--- a/lib/routes/picnob/utils.ts
+++ b/lib/routes/picnob/utils.ts
@@ -1,17 +1,19 @@
-const puppeteerGet = async (url, browser) => {
+import { getPuppeteerPage } from '@/utils/puppeteer';
+
+const puppeteerGet = async (url) => {
let data;
- const page = await browser.newPage();
- await page.setRequestInterception(true);
- page.on('request', (request) => {
- request.resourceType() === 'document' ? request.continue() : request.abort();
+ const { destory } = await getPuppeteerPage(url, {
+ onBeforeLoad: async (page) => {
+ await page.setRequestInterception(true);
+ page.on('request', (request) => {
+ request.resourceType() === 'document' ? request.continue() : request.abort();
+ });
+ page.on('response', async (response) => {
+ data = await (response.request().url().includes('/api/posts') ? response.json() : response.text());
+ });
+ },
});
- page.on('response', async (response) => {
- data = await (response.request().url().includes('/api/posts') ? response.json() : response.text());
- });
- await page.goto(url, {
- waitUntil: 'domcontentloaded',
- });
- await page.close();
+ await destory();
return data;
};
diff --git a/lib/utils/puppeteer.test.ts b/lib/utils/puppeteer.test.ts
index 1fd8f1d5a..e276cb7c7 100644
--- a/lib/utils/puppeteer.test.ts
+++ b/lib/utils/puppeteer.test.ts
@@ -142,3 +142,122 @@ describe('puppeteer', () => {
expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=socks5:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true);
}, 10000);
});
+
+describe('getPuppeteerPage', () => {
+ it('puppeteer run', async () => {
+ const { getPuppeteerPage } = await import('./puppeteer');
+ const pup = await getPuppeteerPage('https://www.google.com');
+ const page = pup.page;
+ browser = pup.browser;
+ const startTime = Date.now();
+
+ const html = await page.evaluate(() => document.body.innerHTML);
+ expect(html.length).toBeGreaterThan(0);
+
+ expect(browser.process()?.exitCode).toBe(null); // browser is still running
+ const sleepTime = 31 * 1000 - (Date.now() - startTime); // prevent long loading time from failing the test
+ if (sleepTime > 0) {
+ await wait(sleepTime);
+ }
+ expect(browser.process()?.exitCode).toBe(0); // browser is closed
+ }, 45000);
+
+ it('puppeteer accept http proxy uri w/ auth', async () => {
+ process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333';
+
+ const { getPuppeteerPage } = await import('./puppeteer');
+ const pup = await getPuppeteerPage('https://www.google.com');
+ browser = pup.browser;
+
+ // trailing slash will cause net::ERR_NO_SUPPORTED_PROXIES, prohibit it
+ expect(browser.process()?.spawnargs.includes('--proxy-server=http://rsshub.proxy:2333')).toBe(true);
+ });
+
+ it('puppeteer respect proxy regex', async () => {
+ process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333';
+ process.env.PROXY_URL_REGEX = 'not-exist';
+
+ const { getPuppeteerPage } = await import('./puppeteer');
+ const pup = await getPuppeteerPage('https://www.google.com');
+ browser = pup.browser;
+
+ // trailing slash will cause net::ERR_NO_SUPPORTED_PROXIES, prohibit it
+ expect(browser.process()?.spawnargs.includes('--proxy-server=http://rsshub.proxy:2333')).toBe(false);
+ });
+
+ it('puppeteer reject https proxy uri w/ auth', async () => {
+ process.env.PROXY_URI = 'https://user:pass@rsshub.proxy:2333';
+
+ const { getPuppeteerPage } = await import('./puppeteer');
+ const pup = await getPuppeteerPage('https://www.google.com');
+ browser = pup.browser;
+
+ expect(browser.process()?.spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false);
+ });
+
+ it('puppeteer reject socks proxy uri w/ auth', async () => {
+ process.env.PROXY_URI = 'socks5://user:pass@rsshub.proxy:2333';
+
+ const { getPuppeteerPage } = await import('./puppeteer');
+ const pup = await getPuppeteerPage('https://www.google.com');
+ browser = pup.browser;
+
+ expect(browser.process()?.spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false);
+ });
+
+ it('puppeteer accept http proxy', async () => {
+ process.env.PROXY_PROTOCOL = 'http';
+ process.env.PROXY_HOST = 'rsshub.proxy';
+ process.env.PROXY_PORT = '2333';
+
+ const { getPuppeteerPage } = await import('./puppeteer');
+ const pup = await getPuppeteerPage('https://www.google.com', {
+ noGoto: true,
+ });
+ browser = pup.browser;
+
+ expect(browser.process()?.spawnargs.includes('--proxy-server=http://rsshub.proxy:2333')).toBe(true);
+ }, 10000);
+
+ it('puppeteer accept https proxy', async () => {
+ process.env.PROXY_PROTOCOL = 'https';
+ process.env.PROXY_HOST = 'rsshub.proxy';
+ process.env.PROXY_PORT = '2333';
+
+ const { getPuppeteerPage } = await import('./puppeteer');
+ const pup = await getPuppeteerPage('https://www.google.com', {
+ noGoto: true,
+ });
+ browser = pup.browser;
+
+ expect(browser.process()?.spawnargs.includes('--proxy-server=https://rsshub.proxy:2333')).toBe(true);
+ }, 10000);
+
+ it('puppeteer accept socks4a proxy', async () => {
+ process.env.PROXY_PROTOCOL = 'socks4a';
+ process.env.PROXY_HOST = 'rsshub.proxy';
+ process.env.PROXY_PORT = '2333';
+
+ const { getPuppeteerPage } = await import('./puppeteer');
+ const pup = await getPuppeteerPage('https://www.google.com', {
+ noGoto: true,
+ });
+ browser = pup.browser;
+
+ expect(browser.process()?.spawnargs.includes('--proxy-server=socks4://rsshub.proxy:2333')).toBe(true);
+ }, 10000);
+
+ it('puppeteer accept socks5h proxy', async () => {
+ process.env.PROXY_PROTOCOL = 'socks5h';
+ process.env.PROXY_HOST = 'rsshub.proxy';
+ process.env.PROXY_PORT = '2333';
+
+ const { getPuppeteerPage } = await import('./puppeteer');
+ const pup = await getPuppeteerPage('https://www.google.com', {
+ noGoto: true,
+ });
+ browser = pup.browser;
+
+ expect(browser.process()?.spawnargs.includes('--proxy-server=socks5://rsshub.proxy:2333')).toBe(true);
+ }, 10000);
+});
diff --git a/lib/utils/puppeteer.ts b/lib/utils/puppeteer.ts
index 2d97ef96b..99aa591dc 100644
--- a/lib/utils/puppeteer.ts
+++ b/lib/utils/puppeteer.ts
@@ -1,5 +1,5 @@
import { config } from '@/config';
-import puppeteer from 'puppeteer';
+import puppeteer, { Browser, Page } from 'puppeteer';
import logger from './logger';
import proxy from './proxy';
import { anonymizeProxy } from 'proxy-chain';
@@ -7,13 +7,8 @@ import { anonymizeProxy } from 'proxy-chain';
import { type PuppeteerExtra, addExtra } from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
-const options = {
- args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
- headless: true,
- ignoreHTTPSErrors: true,
-};
-
/**
+ * @deprecated use getPage instead
* @param {Object} extraOptions
* @param {boolean} extraOptions.stealth - Use puppeteer-extra-plugin-stealth
* @returns Puppeteer browser
@@ -23,13 +18,19 @@ const outPuppeteer = async (
stealth?: boolean;
} = {}
) => {
+ const options = {
+ args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
+ headless: true,
+ ignoreHTTPSErrors: true,
+ };
+
let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer;
if (extraOptions.stealth) {
insidePuppeteer = addExtra(puppeteer);
insidePuppeteer.use(StealthPlugin());
}
- if (proxy.proxyUri) {
+ if (proxy.proxyUri && proxy.proxyObj.url_regex === '.*') {
if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) {
// only proxies with authentication need to be anonymized
if (proxy.proxyUrlHandler.protocol === 'http:') {
@@ -62,3 +63,108 @@ const outPuppeteer = async (
};
export default outPuppeteer;
+
+/**
+ * @returns Puppeteer page
+ */
+export const getPuppeteerPage = async (
+ url: string,
+ instanceOptions: {
+ onBeforeLoad?: (page: Page, browser?: Browser) => Promise | void;
+ gotoConfig?: {
+ waitUntil?: 'load' | 'domcontentloaded' | 'networkidle0';
+ };
+ noGoto?: boolean;
+ } = {}
+) => {
+ const options = {
+ args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
+ headless: true,
+ ignoreHTTPSErrors: true,
+ };
+
+ let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer;
+ insidePuppeteer = addExtra(puppeteer);
+ insidePuppeteer.use(StealthPlugin());
+
+ let allowProxy = false;
+ const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
+ let urlHandler;
+ try {
+ urlHandler = new URL(url);
+ } catch {
+ // ignore
+ }
+
+ if (proxyRegex.test(url) && url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) {
+ allowProxy = true;
+ }
+
+ let hasProxy = false;
+ if (proxy.proxyUri && allowProxy) {
+ if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) {
+ // only proxies with authentication need to be anonymized
+ if (proxy.proxyUrlHandler.protocol === 'http:') {
+ const urlObj = new URL(proxy.proxyUri);
+ urlObj.username = '';
+ urlObj.password = '';
+ options.args.push(`--proxy-server=${urlObj.toString().replace(/\/$/, '')}`);
+ hasProxy = true;
+ } else {
+ logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy');
+ }
+ } else {
+ // Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes
+ options.args.push(`--proxy-server=${proxy.proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
+ hasProxy = true;
+ }
+ }
+ let browser: Browser;
+ if (config.puppeteerWSEndpoint) {
+ const endpointURL = new URL(config.puppeteerWSEndpoint);
+ endpointURL.searchParams.set('launch', JSON.stringify(options));
+ endpointURL.searchParams.set('stealth', 'true');
+ const endpoint = endpointURL.toString();
+ browser = await insidePuppeteer.connect({
+ browserWSEndpoint: endpoint,
+ });
+ } else {
+ browser = await insidePuppeteer.launch(
+ config.chromiumExecutablePath
+ ? {
+ executablePath: config.chromiumExecutablePath,
+ ...options,
+ }
+ : options
+ );
+ }
+
+ setTimeout(async () => {
+ await browser.close();
+ }, 30000);
+
+ const page = await browser.newPage();
+
+ if (hasProxy && (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password)) {
+ await page.authenticate({
+ username: proxy.proxyUrlHandler?.username,
+ password: proxy.proxyUrlHandler?.password,
+ });
+ }
+
+ if (instanceOptions.onBeforeLoad) {
+ await instanceOptions.onBeforeLoad(page, browser);
+ }
+
+ if (!instanceOptions.noGoto) {
+ await page.goto(url, instanceOptions.gotoConfig || { waitUntil: 'domcontentloaded' });
+ }
+
+ return {
+ page,
+ destory: async () => {
+ await browser.close();
+ },
+ browser,
+ };
+};