fix(route/picnob): use puppeteer-real-browser to pass cf check (#20478)

This commit is contained in:
Stephen Zhou
2025-11-14 15:32:09 +08:00
committed by GitHub
parent a07565d356
commit 400fb1fa49
5 changed files with 143 additions and 55 deletions
+1 -1
View File
@@ -148,7 +148,7 @@ RUN \
; \
else \
apt-get install -yq --no-install-recommends \
chromium \
chromium xvfb \
&& \
echo "CHROMIUM_EXECUTABLE_PATH=$(which chromium)" | tee /app/.env ; \
fi; \
+18 -6
View File
@@ -6,14 +6,15 @@ services:
image: diygod/rsshub # or ghcr.io/diygod/rsshub
restart: always
ports:
- "1200:1200"
- '1200:1200'
environment:
NODE_ENV: production
CACHE_TYPE: redis
REDIS_URL: "redis://redis:6379/"
PUPPETEER_WS_ENDPOINT: "ws://browserless:3000" # marked
REDIS_URL: 'redis://redis:6379/'
PUPPETEER_WS_ENDPOINT: 'ws://browserless:3000' # marked
PUPPETEER_REAL_BROWSER_SERVICE: 'http://real-browser:3000' # marked
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:1200/healthz"]
test: ['CMD', 'curl', '-f', 'http://localhost:1200/healthz']
interval: 30s
timeout: 10s
retries: 3
@@ -21,6 +22,17 @@ services:
- redis
- browserless # marked
real-browser:
image: ghcr.io/hyoban/puppeteer-real-browser-hono
restart: always
ports:
- '3001:3000'
healthcheck:
test: ['CMD', 'curl', '-f', 'http://localhost:3000']
interval: 30s
timeout: 10s
retries: 3
browserless: # marked
image: browserless/chrome # marked
restart: always # marked
@@ -29,7 +41,7 @@ services:
hard: 0 # marked
soft: 0 # marked
healthcheck: # marked
test: ["CMD", "curl", "-f", "http://localhost:3000/pressure"] # marked
test: ['CMD', 'curl', '-f', 'http://localhost:3000/pressure'] # marked
interval: 30s # marked
timeout: 10s # marked
retries: 3 # marked
@@ -40,7 +52,7 @@ services:
volumes:
- redis-data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
test: ['CMD', 'redis-cli', 'ping']
interval: 30s
timeout: 10s
retries: 5
+2
View File
@@ -9,6 +9,7 @@ export type Config = {
enableCluster?: string;
isPackage: boolean;
nodeName?: string;
puppeteerRealBrowserService?: string;
puppeteerWSEndpoint?: string;
chromiumExecutablePath?: string;
// network
@@ -480,6 +481,7 @@ const calculateValue = () => {
enableCluster: toBoolean(envs.ENABLE_CLUSTER, false),
isPackage: !!envs.IS_PACKAGE,
nodeName: envs.NODE_NAME,
puppeteerRealBrowserService: envs.PUPPETEER_REAL_BROWSER_SERVICE,
puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT,
chromiumExecutablePath: envs.CHROMIUM_EXECUTABLE_PATH,
// network
+118 -48
View File
@@ -1,11 +1,50 @@
import { config } from '@/config';
import { Route, ViewType } from '@/types';
import ofetch from '@/utils/ofetch';
import { load } from 'cheerio';
import { parseRelativeDate } from '@/utils/parse-date';
import { puppeteerGet } from './utils';
import sanitizeHtml from 'sanitize-html';
import cache from '@/utils/cache';
import { parseRelativeDate } from '@/utils/parse-date';
import { load } from 'cheerio';
import { connect, Options } from 'puppeteer-real-browser';
const realBrowserOption: Options = {
args: ['--start-maximized'],
turnstile: true,
headless: false,
// disableXvfb: true,
// ignoreAllFlags:true,
customConfig: {
chromePath: config.chromiumExecutablePath,
},
connectOption: {
defaultViewport: null,
},
plugins: [],
};
async function getPageWithPuppeteer(url: string, selector: string): Promise<string> {
if (config.puppeteerRealBrowserService) {
const res = await fetch(`${config.puppeteerRealBrowserService}?url=${encodeURIComponent(url)}&selector=${encodeURIComponent(selector)}`);
const json = await res.json();
return (json.data.at(0) || '') as string;
} else {
const { page, browser } = await connect(realBrowserOption);
await page.goto(url, { timeout: 50000 });
let verify: boolean | null = null;
const startDate = Date.now();
while (!verify && Date.now() - startDate < 50000) {
// eslint-disable-next-line no-await-in-loop, no-restricted-syntax
verify = await page.evaluate((sel) => (document.querySelector(sel) ? true : null), selector).catch(() => null);
// eslint-disable-next-line no-await-in-loop
await new Promise((r) => setTimeout(r, 1000));
}
const html = await page.content();
await browser.close();
return html;
}
}
function getProfilePage(profileUrl: string): Promise<string> {
return getPageWithPuppeteer(profileUrl, '.post_box');
}
export const route: Route = {
path: '/user/:id/:type?',
@@ -34,76 +73,109 @@ export const route: Route = {
},
],
name: 'User Profile - Picnob',
maintainers: ['TonyRL', 'micheal-death', 'AiraNadih', 'DIYgod'],
maintainers: ['TonyRL', 'micheal-death', 'AiraNadih', 'DIYgod', 'hyoban'],
handler,
view: ViewType.Pictures,
};
async function handler(ctx) {
if (!config.puppeteerRealBrowserService && !config.chromiumExecutablePath) {
throw new Error('PUPPETEER_REAL_BROWSER_SERVICE or CHROMIUM_EXECUTABLE_PATH is required to use this route.');
}
// NOTE: 'picnob' is still available, but all requests to 'picnob' will be redirected to 'pixnoy' eventually
const baseUrl = 'https://www.pixnoy.com';
const id = ctx.req.param('id');
const type = ctx.req.param('type') ?? 'profile';
const profileUrl = `${baseUrl}/profile/${id}/${type === 'tagged' ? 'tagged/' : ''}`;
// TODO: can't bypass cloudflare 403 error without puppeteer
let html;
let usePuppeteer = false;
try {
const data = await ofetch(profileUrl);
html = data;
} catch {
html = await puppeteerGet(profileUrl);
usePuppeteer = true;
}
const html = await getProfilePage(profileUrl);
const $ = load(html);
const list = $('.post_box')
.toArray()
.map((item) => {
const $item = $(item);
const sum = $item.find('.sum').text();
const coverLink = $item.find('.cover_link').attr('href');
const shortcode = coverLink?.split('/')?.[2];
const image = $item.find('.cover .cover_link img');
const title = image.attr('alt') || '';
return {
title: sanitizeHtml(sum.split('\n')[0], { allowedTags: [], allowedAttributes: {} }),
description: `<img src="${$item.find('.preview_w img').attr('data-src')}" /><br />${sum.replaceAll('\n', '<br>')}`,
title,
description: `<img src="${image.attr('data-src')}" /><br />${title}`,
link: `${baseUrl}${coverLink}`,
guid: shortcode,
pubDate: parseRelativeDate($item.find('.time .txt').text()),
};
});
const newDescription = await Promise.all(
list.map((item) =>
cache.tryGet(`picnob:user:${id}:${item.guid}`, async () => {
try {
const html = usePuppeteer
? await puppeteerGet(item.link)
: await ofetch(item.link, {
headers: {
'user-agent': 'PostmanRuntime/7.44.0',
},
});
const $ = load(html);
if ($('.video_img').length > 0) {
return `<video src="${$('.video_img a').attr('href')}" poster="${$('.video_img img').attr('data-src')}"></video><br />${$('.sum_full').text()}`;
} else {
let description = '';
for (const slide of $('.swiper-slide').toArray()) {
const $slide = $(slide);
description += `<img src="${$slide.find('.pic img').attr('data-src')}" /><br />`;
// Fetch all post details concurrently
// First, get HTML for all posts
let htmlList: string[];
if (config.puppeteerRealBrowserService) {
// Use puppeteer service for multiple URLs
htmlList = (await Promise.all(
list.map((item) =>
cache.tryGet(`picnob:user:${id}:${item.guid}:html`, async () => {
const selector = '.video_img, .swiper-slide';
const res = await fetch(`${config.puppeteerRealBrowserService}?url=${encodeURIComponent(item.link)}&selector=${encodeURIComponent(selector)}`);
const json = await res.json();
return (json.data?.at(0) || '') as string;
})
)
)) as string[];
} else {
// Use local puppeteer browser
const { browser } = await connect(realBrowserOption);
try {
htmlList = (await Promise.all(
list.map((item) =>
cache.tryGet(`picnob:user:${id}:${item.guid}:html`, async () => {
const page = await browser.newPage();
try {
await page.goto(item.link, { timeout: 50000 });
let verify: boolean | null = null;
const startDate = Date.now();
while (!verify && Date.now() - startDate < 50000) {
// eslint-disable-next-line no-await-in-loop, no-restricted-syntax
verify = await page.evaluate(() => (document.querySelector('.video_img') || document.querySelector('.swiper-slide') ? true : null)).catch(() => null);
// eslint-disable-next-line no-await-in-loop
await new Promise((r) => setTimeout(r, 1000));
}
return await page.content();
} catch {
return '';
} finally {
await page.close();
}
description += $('.sum_full').text();
return description;
}
} catch {
return '';
}
})
)
);
})
)
)) as string[];
} finally {
await browser.close();
}
}
// Process HTML to generate descriptions
const newDescription = htmlList.map((html) => {
if (!html) {
return '';
}
const $ = load(html);
if ($('.video_img').length > 0) {
return `<video src="${$('.video_img a').attr('href')}" poster="${$('.video_img img').attr('data-src')}"></video><br />${$('.sum_full').text()}`;
} else {
let description = '';
for (const slide of $('.swiper-slide').toArray()) {
const $slide = $(slide);
description += `<img src="${$slide.find('.pic img').attr('data-src')}" /><br />`;
}
description += $('.sum_full').text();
return description;