From 6288f06b97609d48094ce209d7ccfebc5fd2c9aa Mon Sep 17 00:00:00 2001 From: MMA <58857539+TheGeeKing@users.noreply.github.com> Date: Fri, 3 Apr 2026 23:00:09 +0200 Subject: [PATCH] fix(route/dailypush): use puppeteer (#21160) * use puppeteer * fix typo * fix fetch page one by one * use deprecated method * bypass Disallow await inside of loops * use one browser for the whole route * bypass Disallow await inside of loops * use one browser even if marked deprecated by doc * use Promise.all() * fix new destory -> destroy --- lib/routes/bilibili/cache.ts | 4 +- lib/routes/cjlu/yjsy/index.ts | 4 +- lib/routes/dailypush/all.ts | 32 +++++++++------- lib/routes/dailypush/tags.ts | 32 +++++++++------- lib/routes/dailypush/utils.ts | 59 ++++++++++++++++++++++-------- lib/routes/iwara/ranking.ts | 4 +- lib/routes/iwara/subscriptions.ts | 18 ++++++--- lib/routes/nhentai/util.tsx | 4 +- lib/routes/perplexity/blog.ts | 8 ++-- lib/routes/perplexity/changelog.ts | 4 +- lib/routes/picnob/utils.ts | 4 +- lib/routes/picuki/profile.ts | 4 +- lib/routes/weibo/utils.ts | 4 +- lib/routes/xiaohongshu/util.ts | 4 +- lib/utils/puppeteer.mock.test.ts | 2 +- lib/utils/puppeteer.ts | 2 +- lib/utils/puppeteer.worker.ts | 2 +- 17 files changed, 119 insertions(+), 72 deletions(-) diff --git a/lib/routes/bilibili/cache.ts b/lib/routes/bilibili/cache.ts index 63c3a201f..e4add3e22 100644 --- a/lib/routes/bilibili/cache.ts +++ b/lib/routes/bilibili/cache.ts @@ -38,7 +38,7 @@ const getCookie = (disableConfig = false) => { let waitForRequest = new Promise((resolve) => { resolve(''); }); - const { destory } = await getPuppeteerPage('https://space.bilibili.com/1/dynamic', { + const { destroy } = await getPuppeteerPage('https://space.bilibili.com/1/dynamic', { onBeforeLoad: (page) => { waitForRequest = new Promise((resolve) => { page.on('requestfinished', async (request) => { @@ -54,7 +54,7 @@ const getCookie = (disableConfig = false) => { }); const cookieString = await waitForRequest; logger.debug(`Got bilibili cookie: ${cookieString}`); - await destory(); + await destroy(); return cookieString; }); }; diff --git a/lib/routes/cjlu/yjsy/index.ts b/lib/routes/cjlu/yjsy/index.ts index 1b333b623..894e488bf 100644 --- a/lib/routes/cjlu/yjsy/index.ts +++ b/lib/routes/cjlu/yjsy/index.ts @@ -86,7 +86,7 @@ async function handler(ctx) { const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 10; const url = `${host}index/${cate}.htm`; - const { page, destory, browser } = await getPuppeteerPage(url, { + const { page, destroy, browser } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { await page.setExtraHTTPHeaders(headers); await page.setUserAgent(headers['User-Agent']); @@ -102,7 +102,7 @@ async function handler(ctx) { const cookieString = cookies.map((c) => `${c.name}=${c.value}`).join('; '); const response = await page.content(); - await destory(); + await destroy(); const $ = load(response); diff --git a/lib/routes/dailypush/all.ts b/lib/routes/dailypush/all.ts index 8e0ed6fa3..d15bb923d 100644 --- a/lib/routes/dailypush/all.ts +++ b/lib/routes/dailypush/all.ts @@ -1,9 +1,9 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; -import ofetch from '@/utils/ofetch'; +import puppeteer from '@/utils/puppeteer'; -import { BASE_URL, enhanceItemsWithSummaries, parseArticles } from './utils'; +import { BASE_URL, enhanceItemsWithSummaries, fetchPageHtml, parseArticles } from './utils'; export const route: Route = { path: '/:sort?', @@ -21,7 +21,7 @@ export const route: Route = { }, features: { requireConfig: false, - requirePuppeteer: false, + requirePuppeteer: true, antiCrawler: false, supportBT: false, supportPodcast: false, @@ -42,17 +42,21 @@ async function handler(ctx) { const { sort = '' } = ctx.req.param(); const url = sort ? `${BASE_URL}/${sort}` : BASE_URL; - const response = await ofetch(url); - const $ = load(response); + const browser = await puppeteer(); + try { + const html = await fetchPageHtml(browser, url, 'article'); + const $ = load(html); + const list = parseArticles($, BASE_URL); + const items = await enhanceItemsWithSummaries(browser, list); - const list = parseArticles($, BASE_URL); - const items = await enhanceItemsWithSummaries(list); + const pageTitle = $('title').text() || 'DailyPush - All'; - const pageTitle = $('title').text() || 'DailyPush - All'; - - return { - title: pageTitle, - link: url, - item: items, - }; + return { + title: pageTitle, + link: url, + item: items, + }; + } finally { + await browser.close(); + } } diff --git a/lib/routes/dailypush/tags.ts b/lib/routes/dailypush/tags.ts index 1c4e8bf58..da50a1556 100644 --- a/lib/routes/dailypush/tags.ts +++ b/lib/routes/dailypush/tags.ts @@ -1,9 +1,9 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; -import ofetch from '@/utils/ofetch'; +import puppeteer from '@/utils/puppeteer'; -import { BASE_URL, enhanceItemsWithSummaries, parseArticles } from './utils'; +import { BASE_URL, enhanceItemsWithSummaries, fetchPageHtml, parseArticles } from './utils'; export const route: Route = { path: '/tag/:tag/:sort?', @@ -22,7 +22,7 @@ export const route: Route = { }, features: { requireConfig: false, - requirePuppeteer: false, + requirePuppeteer: true, antiCrawler: false, supportBT: false, supportPodcast: false, @@ -43,17 +43,21 @@ async function handler(ctx) { const { tag, sort = 'trending' } = ctx.req.param(); const url = `${BASE_URL}/${tag}/${sort}`; - const response = await ofetch(url); - const $ = load(response); + const browser = await puppeteer(); + try { + const html = await fetchPageHtml(browser, url, 'article'); + const $ = load(html); + const list = parseArticles($, BASE_URL); + const items = await enhanceItemsWithSummaries(browser, list); - const list = parseArticles($, BASE_URL); - const items = await enhanceItemsWithSummaries(list); + const pageTitle = $('title').text() || `DailyPush - ${tag.charAt(0).toUpperCase() + tag.slice(1)}`; - const pageTitle = $('title').text() || `DailyPush - ${tag.charAt(0).toUpperCase() + tag.slice(1)}`; - - return { - title: pageTitle, - link: url, - item: items, - }; + return { + title: pageTitle, + link: url, + item: items, + }; + } finally { + await browser.close(); + } } diff --git a/lib/routes/dailypush/utils.ts b/lib/routes/dailypush/utils.ts index 041eaa600..2854207ea 100644 --- a/lib/routes/dailypush/utils.ts +++ b/lib/routes/dailypush/utils.ts @@ -1,9 +1,10 @@ import type { CheerioAPI } from 'cheerio'; import { load } from 'cheerio'; +import type { Browser, Page } from 'rebrowser-puppeteer'; import type { DataItem } from '@/types'; import cache from '@/utils/cache'; -import ofetch from '@/utils/ofetch'; +import logger from '@/utils/logger'; import { parseRelativeDate } from '@/utils/parse-date'; export const BASE_URL = 'https://www.dailypush.dev'; @@ -19,6 +20,38 @@ export interface ArticleItem { dailyPushUrl?: string; } +const allowedRequestTypes = new Set(['document']); + +async function preparePage(page: Page) { + await page.setRequestInterception(true); + page.on('request', (request) => { + if (allowedRequestTypes.has(request.resourceType())) { + request.continue(); + return; + } + + request.abort(); + }); +} + +export async function fetchPageHtml(browser: Browser, url: string, waitForSelector?: string): Promise { + const page = await browser.newPage(); + await preparePage(page); + + try { + logger.http(`Requesting ${url}`); + await page.goto(url, { waitUntil: 'domcontentloaded' }); + + if (waitForSelector) { + await page.waitForSelector(waitForSelector); + } + + return await page.content(); + } finally { + await page.close(); + } +} + /** * Try to parse text as a date. Returns the Date if parsing succeeds and is valid, undefined otherwise. */ @@ -40,14 +73,14 @@ function extractAuthor(article: ReturnType): DataItem['author'] { return undefined; } - // Get all content spans (exclude separator spans with '•') + // Get all content spans (exclude separator spans with "•") const allSpans = container.find('span'); const contentSpans: string[] = []; for (let i = 0; i < allSpans.length; i++) { const $span = allSpans.eq(i); const text = $span.text().trim(); - // Skip separator spans (contain only '•' or have separator classes) + // Skip separator spans (contain only "•" or have separator classes) if (text !== '•' && !$span.hasClass('text-slate-300') && !$span.hasClass('dark:text-slate-600')) { contentSpans.push(text); } @@ -127,14 +160,14 @@ function extractPubDate(article: ReturnType): Date | undefined { return undefined; } - // Get all content spans (exclude separator spans with '•') + // Get all content spans (exclude separator spans with "•") const allSpans = container.find('span'); const contentSpans: string[] = []; for (let i = 0; i < allSpans.length; i++) { const $span = allSpans.eq(i); const text = $span.text().trim(); - // Skip separator spans (contain only '•' or have separator classes) + // Skip separator spans (contain only "•" or have separator classes) if (text !== '•' && !$span.hasClass('text-slate-300') && !$span.hasClass('dark:text-slate-600')) { contentSpans.push(text); } @@ -225,23 +258,20 @@ export function parseArticles($: CheerioAPI, baseUrl: string): ArticleItem[] { } /** - * Enhance items with full summaries from dailypush article pages + * Enhance items with full summaries from dailypush article pages. + * Uses the provided browser; opens a new tab per URL (document requests only). Caller must close the browser. */ -export async function enhanceItemsWithSummaries(items: ArticleItem[]): Promise { +export async function enhanceItemsWithSummaries(browser: Browser, items: ArticleItem[]): Promise { const itemsWithUrl = items.filter((item) => item.dailyPushUrl !== undefined); const itemsWithoutUrl: DataItem[] = items.filter((item) => item.dailyPushUrl === undefined); - const enhancedItems: DataItem[] = await Promise.all( + const enhancedItems = await Promise.all( itemsWithUrl.map((item) => cache.tryGet(item.dailyPushUrl!, async () => { - // If we have a dailypush article URL, fetch it for the longer summary try { - const articleResponse = await ofetch(item.dailyPushUrl!); - const $ = load(articleResponse); - - // Find the longer summary/description on the article page + const html = await fetchPageHtml(browser, item.dailyPushUrl!, 'p.font-ibm-plex-sans.leading-relaxed'); + const $ = load(html); const summary = $('p.font-ibm-plex-sans.leading-relaxed').first(); - if (summary.length > 0 && summary.text().trim()) { item.description = summary.text().trim(); } @@ -254,6 +284,5 @@ export async function enhanceItemsWithSummaries(items: ArticleItem[]): Promise { - const { page, destory } = await getPuppeteerPage(url, { + const { page, destroy } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { await page.setRequestInterception(true); page.on('request', (request) => { @@ -83,7 +83,7 @@ async function handler(ctx) { pubDate: parseDate(item.createdAt), })); } finally { - await destory(); + await destroy(); } }, config.cache.routeExpire, diff --git a/lib/routes/iwara/subscriptions.ts b/lib/routes/iwara/subscriptions.ts index 77b058370..3f44add67 100644 --- a/lib/routes/iwara/subscriptions.ts +++ b/lib/routes/iwara/subscriptions.ts @@ -69,7 +69,7 @@ async function handler() { const username = config.iwara.username; const password = config.iwara.password; - const { page, destory } = await getPuppeteerPage(rootUrl, { + const { page, destroy } = await getPuppeteerPage(rootUrl, { gotoConfig: { waitUntil: 'domcontentloaded', }, @@ -113,7 +113,10 @@ async function handler() { async () => { const result = await fetchApi(`${apiqRootUrl}/user/token`, { method: 'POST', - headers: { ...apiHeaders, Authorization: refreshHeaders.authorization }, + headers: { + ...apiHeaders, + Authorization: refreshHeaders.authorization, + }, }); return { authorization: 'Bearer ' + result.accessToken }; }, @@ -121,7 +124,10 @@ async function handler() { false ); - const authedHeaders = { ...apiHeaders, Authorization: authHeaders.authorization }; + const authedHeaders = { + ...apiHeaders, + Authorization: authHeaders.authorization, + }; // fetch subscriptions const [videoResponse, imageResponse] = await Promise.all([ @@ -177,7 +183,9 @@ async function handler() { } const apiUrl = item.link.replace('www.iwara.tv', 'apiq.iwara.tv'); - const response = await fetchApi(apiUrl, { headers: authedHeaders }); + const response = await fetchApi(apiUrl, { + headers: authedHeaders, + }); description = renderSubscriptionImages(response.files ? response.files.filter((f) => f.type === 'image').map((f) => `${imageRootUrl}/image/original/${f.id}/${f.name}`) : [item.imageUrl]); @@ -202,6 +210,6 @@ async function handler() { item: items, }; } finally { - await destory(); + await destroy(); } } diff --git a/lib/routes/nhentai/util.tsx b/lib/routes/nhentai/util.tsx index 16f84cd23..50fc28432 100644 --- a/lib/routes/nhentai/util.tsx +++ b/lib/routes/nhentai/util.tsx @@ -73,7 +73,7 @@ const fetchPage = async (url: string): Promise => { } catch (error: unknown) { const status = (error as { status?: number; statusCode?: number }).status ?? (error as { status?: number; statusCode?: number }).statusCode; if (status === 403) { - const { page, destory } = await getPuppeteerPage(url, { + const { page, destroy } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { const allowedTypes = new Set(['document', 'script', 'xhr', 'fetch']); await page.setRequestInterception(true); @@ -83,7 +83,7 @@ const fetchPage = async (url: string): Promise => { }, }); const content = await page.content(); - await destory(); + await destroy(); return content; } throw error; diff --git a/lib/routes/perplexity/blog.ts b/lib/routes/perplexity/blog.ts index 98f3f61cd..c55ebac1a 100644 --- a/lib/routes/perplexity/blog.ts +++ b/lib/routes/perplexity/blog.ts @@ -38,7 +38,7 @@ async function handler(ctx: Context) { const limit = Number.parseInt(ctx.req.query('limit') ?? '20', 10); const rootUrl = 'https://www.perplexity.ai/hub'; - const { page, destory, browser } = await getPuppeteerPage(rootUrl, { + const { page, destroy, browser } = await getPuppeteerPage(rootUrl, { onBeforeLoad: async (page) => { await page.setRequestInterception(true); page.on('request', (request) => { @@ -119,7 +119,9 @@ async function handler(ctx: Context) { request.resourceType() === 'document' ? request.continue() : request.abort(); }); - await contentPage.goto(item.link!, { waitUntil: 'domcontentloaded' }); + await contentPage.goto(item.link!, { + waitUntil: 'domcontentloaded', + }); const contentHtml = await contentPage.evaluate(() => document.documentElement.innerHTML); await contentPage.close(); @@ -148,7 +150,7 @@ async function handler(ctx: Context) { }) ); - await destory(); + await destroy(); return { title: 'Perplexity Blog', diff --git a/lib/routes/perplexity/changelog.ts b/lib/routes/perplexity/changelog.ts index b6dd5ecd0..9d4698e5e 100644 --- a/lib/routes/perplexity/changelog.ts +++ b/lib/routes/perplexity/changelog.ts @@ -16,7 +16,7 @@ export const handler = async (ctx: Context): Promise => { logger.http(`Fetching Perplexity changelog from ${targetUrl}`); - const { page, destory, browser } = await getPuppeteerPage(targetUrl, { + const { page, destroy, browser } = await getPuppeteerPage(targetUrl, { onBeforeLoad: async (page) => { await page.setRequestInterception(true); page.on('request', (request) => { @@ -131,7 +131,7 @@ export const handler = async (ctx: Context): Promise => { ); // Close the browser session after all requests are done - await destory(); + await destroy(); return { title: $('title').text() || 'Perplexity Changelog', diff --git a/lib/routes/picnob/utils.ts b/lib/routes/picnob/utils.ts index 94cda7c88..1a849f0c2 100644 --- a/lib/routes/picnob/utils.ts +++ b/lib/routes/picnob/utils.ts @@ -2,7 +2,7 @@ import { getPuppeteerPage } from '@/utils/puppeteer'; const puppeteerGet = async (url) => { let data; - const { destory } = await getPuppeteerPage(url, { + const { destroy } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { await page.setRequestInterception(true); page.on('request', (request) => { @@ -13,7 +13,7 @@ const puppeteerGet = async (url) => { }); }, }); - await destory(); + await destroy(); return data; }; diff --git a/lib/routes/picuki/profile.ts b/lib/routes/picuki/profile.ts index 92c1653fd..17aadadc8 100644 --- a/lib/routes/picuki/profile.ts +++ b/lib/routes/picuki/profile.ts @@ -85,7 +85,7 @@ async function handler(ctx) { }); } catch (error) { if (error.status === 403) { - const { page, destory } = await getPuppeteerPage(profileUrl, { + const { page, destroy } = await getPuppeteerPage(profileUrl, { onBeforeLoad: async (page) => { const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']); await page.setRequestInterception(true); @@ -96,7 +96,7 @@ async function handler(ctx) { }); await page.waitForSelector('.content'); response = await page.content(); - await destory(); + await destroy(); } else { throw new NotFoundError(error.message); } diff --git a/lib/routes/weibo/utils.ts b/lib/routes/weibo/utils.ts index 9750dd47b..f33e28315 100644 --- a/lib/routes/weibo/utils.ts +++ b/lib/routes/weibo/utils.ts @@ -80,7 +80,7 @@ const weiboUtils = { logger.info(`Fetching visitor Cookies from ${url}`); } let times = 0; - const { page, destory } = await getPuppeteerPage(url, { + const { page, destroy } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']); await page.setUserAgent(weiboUtils.apiHeaders['User-Agent']); @@ -101,7 +101,7 @@ const weiboUtils = { gotoConfig: { waitUntil: 'networkidle0' }, }); const cookies: string = await getCookies(page, 'weibo.cn'); - await destory(); + await destroy(); if (times < 2 || !cookies) { throw new Error(`Unable to fetch visitor cookies. Please set WEIBO_COOKIES. Redirection: ${times}, last URL: ${page.url()}`); } diff --git a/lib/routes/xiaohongshu/util.ts b/lib/routes/xiaohongshu/util.ts index b4247bea0..dcd23dd56 100644 --- a/lib/routes/xiaohongshu/util.ts +++ b/lib/routes/xiaohongshu/util.ts @@ -63,7 +63,7 @@ const getUser = (url, cache) => } // Use puppeteer - const { page, destory } = await getPuppeteerPage(url, { + const { page, destroy } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { await page.setRequestInterception(true); page.on('request', (request) => { @@ -107,7 +107,7 @@ const getUser = (url, cache) => return { userPageData, notes, collect }; } finally { - await destory(); + await destroy(); } }, config.cache.routeExpire, diff --git a/lib/utils/puppeteer.mock.test.ts b/lib/utils/puppeteer.mock.test.ts index 713347ff3..a4c8e46de 100644 --- a/lib/utils/puppeteer.mock.test.ts +++ b/lib/utils/puppeteer.mock.test.ts @@ -69,7 +69,7 @@ describe('getPuppeteerPage (mocked)', () => { expect(endpoint).toContain('stealth=true'); expect(onBeforeLoad).toHaveBeenCalled(); - await result.destory(); + await result.destroy(); expect(browser.close).toHaveBeenCalled(); delete process.env.PUPPETEER_WS_ENDPOINT; diff --git a/lib/utils/puppeteer.ts b/lib/utils/puppeteer.ts index 1abfbfc77..2888aaf65 100644 --- a/lib/utils/puppeteer.ts +++ b/lib/utils/puppeteer.ts @@ -187,7 +187,7 @@ export const getPuppeteerPage = async ( return { page, - destory: async () => { + destroy: async () => { await browser.close(); }, browser, diff --git a/lib/utils/puppeteer.worker.ts b/lib/utils/puppeteer.worker.ts index 8fcc6efc0..74f94425a 100644 --- a/lib/utils/puppeteer.worker.ts +++ b/lib/utils/puppeteer.worker.ts @@ -91,7 +91,7 @@ export const getPuppeteerPage = async ( return { page, - destory: async () => { + destroy: async () => { await browser.close(); }, browser,