fix: puppeteer proxy regex
This commit is contained in:
DIYgod
parent
09c7f2220a
commit
bf3649324b
|
|
@ -4,7 +4,6 @@ import ofetch from '@/utils/ofetch';
|
|||
import { load } from 'cheerio';
|
||||
import { parseRelativeDate } from '@/utils/parse-date';
|
||||
import { puppeteerGet } from './utils';
|
||||
import puppeteer from '@/utils/puppeteer';
|
||||
import sanitizeHtml from 'sanitize-html';
|
||||
import cache from '@/utils/cache';
|
||||
|
||||
|
|
@ -47,82 +46,73 @@ async function handler(ctx) {
|
|||
const type = ctx.req.param('type') ?? 'profile';
|
||||
const profileUrl = `${baseUrl}/profile/${id}/${type === 'tagged' ? 'tagged/' : ''}`;
|
||||
|
||||
const browser = await puppeteer();
|
||||
// TODO: can't bypass cloudflare 403 error without puppeteer
|
||||
|
||||
let html;
|
||||
let usePuppeteer = false;
|
||||
try {
|
||||
let html;
|
||||
let usePuppeteer = false;
|
||||
try {
|
||||
const data = await ofetch(profileUrl);
|
||||
html = data;
|
||||
} catch {
|
||||
html = await puppeteerGet(profileUrl, browser);
|
||||
usePuppeteer = true;
|
||||
}
|
||||
const $ = load(html);
|
||||
|
||||
const list = $('.post_box')
|
||||
.toArray()
|
||||
.map((item) => {
|
||||
const $item = $(item);
|
||||
const sum = $item.find('.sum').text();
|
||||
const coverLink = $item.find('.cover_link').attr('href');
|
||||
const shortcode = coverLink?.split('/')?.[2];
|
||||
|
||||
return {
|
||||
title: sanitizeHtml(sum.split('\n')[0], { allowedTags: [], allowedAttributes: {} }),
|
||||
description: `<img src="${$item.find('.preview_w img').attr('data-src')}" /><br />${sum.replaceAll('\n', '<br>')}`,
|
||||
link: `${baseUrl}${coverLink}`,
|
||||
guid: shortcode,
|
||||
pubDate: parseRelativeDate($item.find('.time .txt').text()),
|
||||
};
|
||||
});
|
||||
|
||||
const newDescription = await Promise.all(
|
||||
list.map((item) =>
|
||||
cache.tryGet(`picnob:user:${id}:${item.guid}`, async () => {
|
||||
try {
|
||||
const html = usePuppeteer
|
||||
? await puppeteerGet(item.link, browser)
|
||||
: await ofetch(item.link, {
|
||||
headers: {
|
||||
'user-agent': 'PostmanRuntime/7.44.0',
|
||||
},
|
||||
});
|
||||
const $ = load(html);
|
||||
if ($('.video_img').length > 0) {
|
||||
return `<video src="${$('.video_img a').attr('href')}" poster="${$('.video_img img').attr('data-src')}"></video><br />${$('.sum_full').text()}`;
|
||||
} else {
|
||||
let description = '';
|
||||
for (const slide of $('.swiper-slide').toArray()) {
|
||||
const $slide = $(slide);
|
||||
description += `<img src="${$slide.find('.pic img').attr('data-src')}" /><br />`;
|
||||
}
|
||||
description += $('.sum_full').text();
|
||||
return description;
|
||||
}
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
return {
|
||||
title: `${$('h1.fullname').text()} (@${id}) ${type === 'tagged' ? 'tagged' : 'public'} posts - Picnob`,
|
||||
description: $('.info .sum').text(),
|
||||
link: profileUrl,
|
||||
image: $('.ava .pic img').attr('src'),
|
||||
item: list.map((item, index) => ({
|
||||
...item,
|
||||
description: newDescription[index] || item.description,
|
||||
})),
|
||||
};
|
||||
} catch (error) {
|
||||
await browser.close();
|
||||
throw error;
|
||||
} finally {
|
||||
await browser.close();
|
||||
const data = await ofetch(profileUrl);
|
||||
html = data;
|
||||
} catch {
|
||||
html = await puppeteerGet(profileUrl);
|
||||
usePuppeteer = true;
|
||||
}
|
||||
const $ = load(html);
|
||||
|
||||
const list = $('.post_box')
|
||||
.toArray()
|
||||
.map((item) => {
|
||||
const $item = $(item);
|
||||
const sum = $item.find('.sum').text();
|
||||
const coverLink = $item.find('.cover_link').attr('href');
|
||||
const shortcode = coverLink?.split('/')?.[2];
|
||||
|
||||
return {
|
||||
title: sanitizeHtml(sum.split('\n')[0], { allowedTags: [], allowedAttributes: {} }),
|
||||
description: `<img src="${$item.find('.preview_w img').attr('data-src')}" /><br />${sum.replaceAll('\n', '<br>')}`,
|
||||
link: `${baseUrl}${coverLink}`,
|
||||
guid: shortcode,
|
||||
pubDate: parseRelativeDate($item.find('.time .txt').text()),
|
||||
};
|
||||
});
|
||||
|
||||
const newDescription = await Promise.all(
|
||||
list.map((item) =>
|
||||
cache.tryGet(`picnob:user:${id}:${item.guid}`, async () => {
|
||||
try {
|
||||
const html = usePuppeteer
|
||||
? await puppeteerGet(item.link)
|
||||
: await ofetch(item.link, {
|
||||
headers: {
|
||||
'user-agent': 'PostmanRuntime/7.44.0',
|
||||
},
|
||||
});
|
||||
const $ = load(html);
|
||||
if ($('.video_img').length > 0) {
|
||||
return `<video src="${$('.video_img a').attr('href')}" poster="${$('.video_img img').attr('data-src')}"></video><br />${$('.sum_full').text()}`;
|
||||
} else {
|
||||
let description = '';
|
||||
for (const slide of $('.swiper-slide').toArray()) {
|
||||
const $slide = $(slide);
|
||||
description += `<img src="${$slide.find('.pic img').attr('data-src')}" /><br />`;
|
||||
}
|
||||
description += $('.sum_full').text();
|
||||
return description;
|
||||
}
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
return {
|
||||
title: `${$('h1.fullname').text()} (@${id}) ${type === 'tagged' ? 'tagged' : 'public'} posts - Picnob`,
|
||||
description: $('.info .sum').text(),
|
||||
link: profileUrl,
|
||||
image: $('.ava .pic img').attr('src'),
|
||||
item: list.map((item, index) => ({
|
||||
...item,
|
||||
description: newDescription[index] || item.description,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,19 @@
|
|||
const puppeteerGet = async (url, browser) => {
|
||||
import { getPuppeteerPage } from '@/utils/puppeteer';
|
||||
|
||||
const puppeteerGet = async (url) => {
|
||||
let data;
|
||||
const page = await browser.newPage();
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (request) => {
|
||||
request.resourceType() === 'document' ? request.continue() : request.abort();
|
||||
const { destory } = await getPuppeteerPage(url, {
|
||||
onBeforeLoad: async (page) => {
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (request) => {
|
||||
request.resourceType() === 'document' ? request.continue() : request.abort();
|
||||
});
|
||||
page.on('response', async (response) => {
|
||||
data = await (response.request().url().includes('/api/posts') ? response.json() : response.text());
|
||||
});
|
||||
},
|
||||
});
|
||||
page.on('response', async (response) => {
|
||||
data = await (response.request().url().includes('/api/posts') ? response.json() : response.text());
|
||||
});
|
||||
await page.goto(url, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
});
|
||||
await page.close();
|
||||
await destory();
|
||||
return data;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -142,3 +142,122 @@ describe('puppeteer', () => {
|
|||
expect(browser.process()?.spawnargs.some((arg) => /^--proxy-server=socks5:\/\/rsshub.proxy:2333$/.test(arg))).toBe(true);
|
||||
}, 10000);
|
||||
});
|
||||
|
||||
describe('getPuppeteerPage', () => {
|
||||
it('puppeteer run', async () => {
|
||||
const { getPuppeteerPage } = await import('./puppeteer');
|
||||
const pup = await getPuppeteerPage('https://www.google.com');
|
||||
const page = pup.page;
|
||||
browser = pup.browser;
|
||||
const startTime = Date.now();
|
||||
|
||||
const html = await page.evaluate(() => document.body.innerHTML);
|
||||
expect(html.length).toBeGreaterThan(0);
|
||||
|
||||
expect(browser.process()?.exitCode).toBe(null); // browser is still running
|
||||
const sleepTime = 31 * 1000 - (Date.now() - startTime); // prevent long loading time from failing the test
|
||||
if (sleepTime > 0) {
|
||||
await wait(sleepTime);
|
||||
}
|
||||
expect(browser.process()?.exitCode).toBe(0); // browser is closed
|
||||
}, 45000);
|
||||
|
||||
it('puppeteer accept http proxy uri w/ auth', async () => {
|
||||
process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333';
|
||||
|
||||
const { getPuppeteerPage } = await import('./puppeteer');
|
||||
const pup = await getPuppeteerPage('https://www.google.com');
|
||||
browser = pup.browser;
|
||||
|
||||
// trailing slash will cause net::ERR_NO_SUPPORTED_PROXIES, prohibit it
|
||||
expect(browser.process()?.spawnargs.includes('--proxy-server=http://rsshub.proxy:2333')).toBe(true);
|
||||
});
|
||||
|
||||
it('puppeteer respect proxy regex', async () => {
|
||||
process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333';
|
||||
process.env.PROXY_URL_REGEX = 'not-exist';
|
||||
|
||||
const { getPuppeteerPage } = await import('./puppeteer');
|
||||
const pup = await getPuppeteerPage('https://www.google.com');
|
||||
browser = pup.browser;
|
||||
|
||||
// trailing slash will cause net::ERR_NO_SUPPORTED_PROXIES, prohibit it
|
||||
expect(browser.process()?.spawnargs.includes('--proxy-server=http://rsshub.proxy:2333')).toBe(false);
|
||||
});
|
||||
|
||||
it('puppeteer reject https proxy uri w/ auth', async () => {
|
||||
process.env.PROXY_URI = 'https://user:pass@rsshub.proxy:2333';
|
||||
|
||||
const { getPuppeteerPage } = await import('./puppeteer');
|
||||
const pup = await getPuppeteerPage('https://www.google.com');
|
||||
browser = pup.browser;
|
||||
|
||||
expect(browser.process()?.spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false);
|
||||
});
|
||||
|
||||
it('puppeteer reject socks proxy uri w/ auth', async () => {
|
||||
process.env.PROXY_URI = 'socks5://user:pass@rsshub.proxy:2333';
|
||||
|
||||
const { getPuppeteerPage } = await import('./puppeteer');
|
||||
const pup = await getPuppeteerPage('https://www.google.com');
|
||||
browser = pup.browser;
|
||||
|
||||
expect(browser.process()?.spawnargs.some((arg) => arg.includes('--proxy-server'))).toBe(false);
|
||||
});
|
||||
|
||||
it('puppeteer accept http proxy', async () => {
|
||||
process.env.PROXY_PROTOCOL = 'http';
|
||||
process.env.PROXY_HOST = 'rsshub.proxy';
|
||||
process.env.PROXY_PORT = '2333';
|
||||
|
||||
const { getPuppeteerPage } = await import('./puppeteer');
|
||||
const pup = await getPuppeteerPage('https://www.google.com', {
|
||||
noGoto: true,
|
||||
});
|
||||
browser = pup.browser;
|
||||
|
||||
expect(browser.process()?.spawnargs.includes('--proxy-server=http://rsshub.proxy:2333')).toBe(true);
|
||||
}, 10000);
|
||||
|
||||
it('puppeteer accept https proxy', async () => {
|
||||
process.env.PROXY_PROTOCOL = 'https';
|
||||
process.env.PROXY_HOST = 'rsshub.proxy';
|
||||
process.env.PROXY_PORT = '2333';
|
||||
|
||||
const { getPuppeteerPage } = await import('./puppeteer');
|
||||
const pup = await getPuppeteerPage('https://www.google.com', {
|
||||
noGoto: true,
|
||||
});
|
||||
browser = pup.browser;
|
||||
|
||||
expect(browser.process()?.spawnargs.includes('--proxy-server=https://rsshub.proxy:2333')).toBe(true);
|
||||
}, 10000);
|
||||
|
||||
it('puppeteer accept socks4a proxy', async () => {
|
||||
process.env.PROXY_PROTOCOL = 'socks4a';
|
||||
process.env.PROXY_HOST = 'rsshub.proxy';
|
||||
process.env.PROXY_PORT = '2333';
|
||||
|
||||
const { getPuppeteerPage } = await import('./puppeteer');
|
||||
const pup = await getPuppeteerPage('https://www.google.com', {
|
||||
noGoto: true,
|
||||
});
|
||||
browser = pup.browser;
|
||||
|
||||
expect(browser.process()?.spawnargs.includes('--proxy-server=socks4://rsshub.proxy:2333')).toBe(true);
|
||||
}, 10000);
|
||||
|
||||
it('puppeteer accept socks5h proxy', async () => {
|
||||
process.env.PROXY_PROTOCOL = 'socks5h';
|
||||
process.env.PROXY_HOST = 'rsshub.proxy';
|
||||
process.env.PROXY_PORT = '2333';
|
||||
|
||||
const { getPuppeteerPage } = await import('./puppeteer');
|
||||
const pup = await getPuppeteerPage('https://www.google.com', {
|
||||
noGoto: true,
|
||||
});
|
||||
browser = pup.browser;
|
||||
|
||||
expect(browser.process()?.spawnargs.includes('--proxy-server=socks5://rsshub.proxy:2333')).toBe(true);
|
||||
}, 10000);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { config } from '@/config';
|
||||
import puppeteer from 'puppeteer';
|
||||
import puppeteer, { Browser, Page } from 'puppeteer';
|
||||
import logger from './logger';
|
||||
import proxy from './proxy';
|
||||
import { anonymizeProxy } from 'proxy-chain';
|
||||
|
|
@ -7,13 +7,8 @@ import { anonymizeProxy } from 'proxy-chain';
|
|||
import { type PuppeteerExtra, addExtra } from 'puppeteer-extra';
|
||||
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
||||
|
||||
const options = {
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
|
||||
headless: true,
|
||||
ignoreHTTPSErrors: true,
|
||||
};
|
||||
|
||||
/**
|
||||
* @deprecated use getPage instead
|
||||
* @param {Object} extraOptions
|
||||
* @param {boolean} extraOptions.stealth - Use puppeteer-extra-plugin-stealth
|
||||
* @returns Puppeteer browser
|
||||
|
|
@ -23,13 +18,19 @@ const outPuppeteer = async (
|
|||
stealth?: boolean;
|
||||
} = {}
|
||||
) => {
|
||||
const options = {
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
|
||||
headless: true,
|
||||
ignoreHTTPSErrors: true,
|
||||
};
|
||||
|
||||
let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer;
|
||||
if (extraOptions.stealth) {
|
||||
insidePuppeteer = addExtra(puppeteer);
|
||||
insidePuppeteer.use(StealthPlugin());
|
||||
}
|
||||
|
||||
if (proxy.proxyUri) {
|
||||
if (proxy.proxyUri && proxy.proxyObj.url_regex === '.*') {
|
||||
if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) {
|
||||
// only proxies with authentication need to be anonymized
|
||||
if (proxy.proxyUrlHandler.protocol === 'http:') {
|
||||
|
|
@ -62,3 +63,108 @@ const outPuppeteer = async (
|
|||
};
|
||||
|
||||
export default outPuppeteer;
|
||||
|
||||
/**
|
||||
* @returns Puppeteer page
|
||||
*/
|
||||
export const getPuppeteerPage = async (
|
||||
url: string,
|
||||
instanceOptions: {
|
||||
onBeforeLoad?: (page: Page, browser?: Browser) => Promise<void> | void;
|
||||
gotoConfig?: {
|
||||
waitUntil?: 'load' | 'domcontentloaded' | 'networkidle0';
|
||||
};
|
||||
noGoto?: boolean;
|
||||
} = {}
|
||||
) => {
|
||||
const options = {
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
|
||||
headless: true,
|
||||
ignoreHTTPSErrors: true,
|
||||
};
|
||||
|
||||
let insidePuppeteer: PuppeteerExtra | typeof puppeteer = puppeteer;
|
||||
insidePuppeteer = addExtra(puppeteer);
|
||||
insidePuppeteer.use(StealthPlugin());
|
||||
|
||||
let allowProxy = false;
|
||||
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
|
||||
let urlHandler;
|
||||
try {
|
||||
urlHandler = new URL(url);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
if (proxyRegex.test(url) && url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) {
|
||||
allowProxy = true;
|
||||
}
|
||||
|
||||
let hasProxy = false;
|
||||
if (proxy.proxyUri && allowProxy) {
|
||||
if (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password) {
|
||||
// only proxies with authentication need to be anonymized
|
||||
if (proxy.proxyUrlHandler.protocol === 'http:') {
|
||||
const urlObj = new URL(proxy.proxyUri);
|
||||
urlObj.username = '';
|
||||
urlObj.password = '';
|
||||
options.args.push(`--proxy-server=${urlObj.toString().replace(/\/$/, '')}`);
|
||||
hasProxy = true;
|
||||
} else {
|
||||
logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy');
|
||||
}
|
||||
} else {
|
||||
// Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes
|
||||
options.args.push(`--proxy-server=${proxy.proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
|
||||
hasProxy = true;
|
||||
}
|
||||
}
|
||||
let browser: Browser;
|
||||
if (config.puppeteerWSEndpoint) {
|
||||
const endpointURL = new URL(config.puppeteerWSEndpoint);
|
||||
endpointURL.searchParams.set('launch', JSON.stringify(options));
|
||||
endpointURL.searchParams.set('stealth', 'true');
|
||||
const endpoint = endpointURL.toString();
|
||||
browser = await insidePuppeteer.connect({
|
||||
browserWSEndpoint: endpoint,
|
||||
});
|
||||
} else {
|
||||
browser = await insidePuppeteer.launch(
|
||||
config.chromiumExecutablePath
|
||||
? {
|
||||
executablePath: config.chromiumExecutablePath,
|
||||
...options,
|
||||
}
|
||||
: options
|
||||
);
|
||||
}
|
||||
|
||||
setTimeout(async () => {
|
||||
await browser.close();
|
||||
}, 30000);
|
||||
|
||||
const page = await browser.newPage();
|
||||
|
||||
if (hasProxy && (proxy.proxyUrlHandler?.username || proxy.proxyUrlHandler?.password)) {
|
||||
await page.authenticate({
|
||||
username: proxy.proxyUrlHandler?.username,
|
||||
password: proxy.proxyUrlHandler?.password,
|
||||
});
|
||||
}
|
||||
|
||||
if (instanceOptions.onBeforeLoad) {
|
||||
await instanceOptions.onBeforeLoad(page, browser);
|
||||
}
|
||||
|
||||
if (!instanceOptions.noGoto) {
|
||||
await page.goto(url, instanceOptions.gotoConfig || { waitUntil: 'domcontentloaded' });
|
||||
}
|
||||
|
||||
return {
|
||||
page,
|
||||
destory: async () => {
|
||||
await browser.close();
|
||||
},
|
||||
browser,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue