fix(route/picnob): use puppeteer-real-browser to pass cf check (#20478)
This commit is contained in:
+1
-1
@@ -148,7 +148,7 @@ RUN \
|
||||
; \
|
||||
else \
|
||||
apt-get install -yq --no-install-recommends \
|
||||
chromium \
|
||||
chromium xvfb \
|
||||
&& \
|
||||
echo "CHROMIUM_EXECUTABLE_PATH=$(which chromium)" | tee /app/.env ; \
|
||||
fi; \
|
||||
|
||||
+18
-6
@@ -6,14 +6,15 @@ services:
|
||||
image: diygod/rsshub # or ghcr.io/diygod/rsshub
|
||||
restart: always
|
||||
ports:
|
||||
- "1200:1200"
|
||||
- '1200:1200'
|
||||
environment:
|
||||
NODE_ENV: production
|
||||
CACHE_TYPE: redis
|
||||
REDIS_URL: "redis://redis:6379/"
|
||||
PUPPETEER_WS_ENDPOINT: "ws://browserless:3000" # marked
|
||||
REDIS_URL: 'redis://redis:6379/'
|
||||
PUPPETEER_WS_ENDPOINT: 'ws://browserless:3000' # marked
|
||||
PUPPETEER_REAL_BROWSER_SERVICE: 'http://real-browser:3000' # marked
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:1200/healthz"]
|
||||
test: ['CMD', 'curl', '-f', 'http://localhost:1200/healthz']
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
@@ -21,6 +22,17 @@ services:
|
||||
- redis
|
||||
- browserless # marked
|
||||
|
||||
real-browser:
|
||||
image: ghcr.io/hyoban/puppeteer-real-browser-hono
|
||||
restart: always
|
||||
ports:
|
||||
- '3001:3000'
|
||||
healthcheck:
|
||||
test: ['CMD', 'curl', '-f', 'http://localhost:3000']
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
browserless: # marked
|
||||
image: browserless/chrome # marked
|
||||
restart: always # marked
|
||||
@@ -29,7 +41,7 @@ services:
|
||||
hard: 0 # marked
|
||||
soft: 0 # marked
|
||||
healthcheck: # marked
|
||||
test: ["CMD", "curl", "-f", "http://localhost:3000/pressure"] # marked
|
||||
test: ['CMD', 'curl', '-f', 'http://localhost:3000/pressure'] # marked
|
||||
interval: 30s # marked
|
||||
timeout: 10s # marked
|
||||
retries: 3 # marked
|
||||
@@ -40,7 +52,7 @@ services:
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
test: ['CMD', 'redis-cli', 'ping']
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
@@ -9,6 +9,7 @@ export type Config = {
|
||||
enableCluster?: string;
|
||||
isPackage: boolean;
|
||||
nodeName?: string;
|
||||
puppeteerRealBrowserService?: string;
|
||||
puppeteerWSEndpoint?: string;
|
||||
chromiumExecutablePath?: string;
|
||||
// network
|
||||
@@ -480,6 +481,7 @@ const calculateValue = () => {
|
||||
enableCluster: toBoolean(envs.ENABLE_CLUSTER, false),
|
||||
isPackage: !!envs.IS_PACKAGE,
|
||||
nodeName: envs.NODE_NAME,
|
||||
puppeteerRealBrowserService: envs.PUPPETEER_REAL_BROWSER_SERVICE,
|
||||
puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT,
|
||||
chromiumExecutablePath: envs.CHROMIUM_EXECUTABLE_PATH,
|
||||
// network
|
||||
|
||||
+118
-48
@@ -1,11 +1,50 @@
|
||||
import { config } from '@/config';
|
||||
import { Route, ViewType } from '@/types';
|
||||
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { load } from 'cheerio';
|
||||
import { parseRelativeDate } from '@/utils/parse-date';
|
||||
import { puppeteerGet } from './utils';
|
||||
import sanitizeHtml from 'sanitize-html';
|
||||
import cache from '@/utils/cache';
|
||||
import { parseRelativeDate } from '@/utils/parse-date';
|
||||
import { load } from 'cheerio';
|
||||
import { connect, Options } from 'puppeteer-real-browser';
|
||||
|
||||
const realBrowserOption: Options = {
|
||||
args: ['--start-maximized'],
|
||||
turnstile: true,
|
||||
headless: false,
|
||||
// disableXvfb: true,
|
||||
// ignoreAllFlags:true,
|
||||
customConfig: {
|
||||
chromePath: config.chromiumExecutablePath,
|
||||
},
|
||||
connectOption: {
|
||||
defaultViewport: null,
|
||||
},
|
||||
plugins: [],
|
||||
};
|
||||
|
||||
async function getPageWithPuppeteer(url: string, selector: string): Promise<string> {
|
||||
if (config.puppeteerRealBrowserService) {
|
||||
const res = await fetch(`${config.puppeteerRealBrowserService}?url=${encodeURIComponent(url)}&selector=${encodeURIComponent(selector)}`);
|
||||
const json = await res.json();
|
||||
return (json.data.at(0) || '') as string;
|
||||
} else {
|
||||
const { page, browser } = await connect(realBrowserOption);
|
||||
await page.goto(url, { timeout: 50000 });
|
||||
let verify: boolean | null = null;
|
||||
const startDate = Date.now();
|
||||
while (!verify && Date.now() - startDate < 50000) {
|
||||
// eslint-disable-next-line no-await-in-loop, no-restricted-syntax
|
||||
verify = await page.evaluate((sel) => (document.querySelector(sel) ? true : null), selector).catch(() => null);
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await new Promise((r) => setTimeout(r, 1000));
|
||||
}
|
||||
const html = await page.content();
|
||||
await browser.close();
|
||||
return html;
|
||||
}
|
||||
}
|
||||
|
||||
function getProfilePage(profileUrl: string): Promise<string> {
|
||||
return getPageWithPuppeteer(profileUrl, '.post_box');
|
||||
}
|
||||
|
||||
export const route: Route = {
|
||||
path: '/user/:id/:type?',
|
||||
@@ -34,76 +73,109 @@ export const route: Route = {
|
||||
},
|
||||
],
|
||||
name: 'User Profile - Picnob',
|
||||
maintainers: ['TonyRL', 'micheal-death', 'AiraNadih', 'DIYgod'],
|
||||
maintainers: ['TonyRL', 'micheal-death', 'AiraNadih', 'DIYgod', 'hyoban'],
|
||||
handler,
|
||||
view: ViewType.Pictures,
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
if (!config.puppeteerRealBrowserService && !config.chromiumExecutablePath) {
|
||||
throw new Error('PUPPETEER_REAL_BROWSER_SERVICE or CHROMIUM_EXECUTABLE_PATH is required to use this route.');
|
||||
}
|
||||
|
||||
// NOTE: 'picnob' is still available, but all requests to 'picnob' will be redirected to 'pixnoy' eventually
|
||||
const baseUrl = 'https://www.pixnoy.com';
|
||||
const id = ctx.req.param('id');
|
||||
const type = ctx.req.param('type') ?? 'profile';
|
||||
const profileUrl = `${baseUrl}/profile/${id}/${type === 'tagged' ? 'tagged/' : ''}`;
|
||||
|
||||
// TODO: can't bypass cloudflare 403 error without puppeteer
|
||||
let html;
|
||||
let usePuppeteer = false;
|
||||
try {
|
||||
const data = await ofetch(profileUrl);
|
||||
html = data;
|
||||
} catch {
|
||||
html = await puppeteerGet(profileUrl);
|
||||
usePuppeteer = true;
|
||||
}
|
||||
const html = await getProfilePage(profileUrl);
|
||||
|
||||
const $ = load(html);
|
||||
|
||||
const list = $('.post_box')
|
||||
.toArray()
|
||||
.map((item) => {
|
||||
const $item = $(item);
|
||||
const sum = $item.find('.sum').text();
|
||||
const coverLink = $item.find('.cover_link').attr('href');
|
||||
const shortcode = coverLink?.split('/')?.[2];
|
||||
const image = $item.find('.cover .cover_link img');
|
||||
const title = image.attr('alt') || '';
|
||||
|
||||
return {
|
||||
title: sanitizeHtml(sum.split('\n')[0], { allowedTags: [], allowedAttributes: {} }),
|
||||
description: `<img src="${$item.find('.preview_w img').attr('data-src')}" /><br />${sum.replaceAll('\n', '<br>')}`,
|
||||
title,
|
||||
description: `<img src="${image.attr('data-src')}" /><br />${title}`,
|
||||
link: `${baseUrl}${coverLink}`,
|
||||
guid: shortcode,
|
||||
pubDate: parseRelativeDate($item.find('.time .txt').text()),
|
||||
};
|
||||
});
|
||||
|
||||
const newDescription = await Promise.all(
|
||||
list.map((item) =>
|
||||
cache.tryGet(`picnob:user:${id}:${item.guid}`, async () => {
|
||||
try {
|
||||
const html = usePuppeteer
|
||||
? await puppeteerGet(item.link)
|
||||
: await ofetch(item.link, {
|
||||
headers: {
|
||||
'user-agent': 'PostmanRuntime/7.44.0',
|
||||
},
|
||||
});
|
||||
const $ = load(html);
|
||||
if ($('.video_img').length > 0) {
|
||||
return `<video src="${$('.video_img a').attr('href')}" poster="${$('.video_img img').attr('data-src')}"></video><br />${$('.sum_full').text()}`;
|
||||
} else {
|
||||
let description = '';
|
||||
for (const slide of $('.swiper-slide').toArray()) {
|
||||
const $slide = $(slide);
|
||||
description += `<img src="${$slide.find('.pic img').attr('data-src')}" /><br />`;
|
||||
// Fetch all post details concurrently
|
||||
// First, get HTML for all posts
|
||||
let htmlList: string[];
|
||||
if (config.puppeteerRealBrowserService) {
|
||||
// Use puppeteer service for multiple URLs
|
||||
htmlList = (await Promise.all(
|
||||
list.map((item) =>
|
||||
cache.tryGet(`picnob:user:${id}:${item.guid}:html`, async () => {
|
||||
const selector = '.video_img, .swiper-slide';
|
||||
const res = await fetch(`${config.puppeteerRealBrowserService}?url=${encodeURIComponent(item.link)}&selector=${encodeURIComponent(selector)}`);
|
||||
const json = await res.json();
|
||||
return (json.data?.at(0) || '') as string;
|
||||
})
|
||||
)
|
||||
)) as string[];
|
||||
} else {
|
||||
// Use local puppeteer browser
|
||||
const { browser } = await connect(realBrowserOption);
|
||||
try {
|
||||
htmlList = (await Promise.all(
|
||||
list.map((item) =>
|
||||
cache.tryGet(`picnob:user:${id}:${item.guid}:html`, async () => {
|
||||
const page = await browser.newPage();
|
||||
try {
|
||||
await page.goto(item.link, { timeout: 50000 });
|
||||
let verify: boolean | null = null;
|
||||
const startDate = Date.now();
|
||||
while (!verify && Date.now() - startDate < 50000) {
|
||||
// eslint-disable-next-line no-await-in-loop, no-restricted-syntax
|
||||
verify = await page.evaluate(() => (document.querySelector('.video_img') || document.querySelector('.swiper-slide') ? true : null)).catch(() => null);
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await new Promise((r) => setTimeout(r, 1000));
|
||||
}
|
||||
return await page.content();
|
||||
} catch {
|
||||
return '';
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
description += $('.sum_full').text();
|
||||
return description;
|
||||
}
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
})
|
||||
)
|
||||
);
|
||||
})
|
||||
)
|
||||
)) as string[];
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Process HTML to generate descriptions
|
||||
const newDescription = htmlList.map((html) => {
|
||||
if (!html) {
|
||||
return '';
|
||||
}
|
||||
const $ = load(html);
|
||||
if ($('.video_img').length > 0) {
|
||||
return `<video src="${$('.video_img a').attr('href')}" poster="${$('.video_img img').attr('data-src')}"></video><br />${$('.sum_full').text()}`;
|
||||
} else {
|
||||
let description = '';
|
||||
for (const slide of $('.swiper-slide').toArray()) {
|
||||
const $slide = $(slide);
|
||||
description += `<img src="${$slide.find('.pic img').attr('data-src')}" /><br />`;
|
||||
}
|
||||
description += $('.sum_full').text();
|
||||
return description;
|
||||
|
||||
Reference in New Issue
Block a user