xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(route/dailypush): use puppeteer (#21160) 

* use puppeteer

* fix typo

* fix fetch page one by one

* use deprecated method

* bypass Disallow await inside of loops

* use one browser for the whole route

* bypass Disallow await inside of loops

* use one browser even if marked deprecated by doc

* use Promise.all()

* fix new destory -> destroy
This commit is contained in:
MMA 2026-04-03 23:00:09 +02:00 committed by GitHub
parent de181a6cdc
commit 6288f06b97
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 119 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/routes/bilibili/cache.ts
View File

@ -38,7 +38,7 @@ const getCookie = (disableConfig = false) => {
let waitForRequest = new Promise<string>((resolve) => {
resolve('');
});
const { destory } = await getPuppeteerPage('https://space.bilibili.com/1/dynamic', {
const { destroy } = await getPuppeteerPage('https://space.bilibili.com/1/dynamic', {
onBeforeLoad: (page) => {
waitForRequest = new Promise<string>((resolve) => {
page.on('requestfinished', async (request) => {
@ -54,7 +54,7 @@ const getCookie = (disableConfig = false) => {
});
const cookieString = await waitForRequest;
logger.debug(`Got bilibili cookie: ${cookieString}`);
await destory();
await destroy();
return cookieString;
});
};

4
lib/routes/cjlu/yjsy/index.ts
View File

@ -86,7 +86,7 @@ async function handler(ctx) {
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 10;
const url = `${host}index/${cate}.htm`;
const { page, destory, browser } = await getPuppeteerPage(url, {
const { page, destroy, browser } = await getPuppeteerPage(url, {
onBeforeLoad: async (page) => {
await page.setExtraHTTPHeaders(headers);
await page.setUserAgent(headers['User-Agent']);
@ -102,7 +102,7 @@ async function handler(ctx) {
const cookieString = cookies.map((c) => `${c.name}=${c.value}`).join('; ');
const response = await page.content();
await destory();
await destroy();
const $ = load(response);

32
lib/routes/dailypush/all.ts
View File

@ -1,9 +1,9 @@
import { load } from 'cheerio';
import type { Route } from '@/types';
import ofetch from '@/utils/ofetch';
import puppeteer from '@/utils/puppeteer';
import { BASE_URL, enhanceItemsWithSummaries, parseArticles } from './utils';
import { BASE_URL, enhanceItemsWithSummaries, fetchPageHtml, parseArticles } from './utils';
export const route: Route = {
path: '/:sort?',
@ -21,7 +21,7 @@ export const route: Route = {
},
features: {
requireConfig: false,
requirePuppeteer: false,
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
@ -42,17 +42,21 @@ async function handler(ctx) {
const { sort = '' } = ctx.req.param();
const url = sort ? `${BASE_URL}/${sort}` : BASE_URL;
const response = await ofetch(url);
const $ = load(response);
const browser = await puppeteer();
try {
const html = await fetchPageHtml(browser, url, 'article');
const $ = load(html);
const list = parseArticles($, BASE_URL);
const items = await enhanceItemsWithSummaries(browser, list);
const list = parseArticles($, BASE_URL);
const items = await enhanceItemsWithSummaries(list);
const pageTitle = $('title').text() || 'DailyPush - All';
const pageTitle = $('title').text() || 'DailyPush - All';
return {
title: pageTitle,
link: url,
item: items,
};
return {
title: pageTitle,
link: url,
item: items,
};
} finally {
await browser.close();
}
}

32
lib/routes/dailypush/tags.ts
View File

@ -1,9 +1,9 @@
import { load } from 'cheerio';
import type { Route } from '@/types';
import ofetch from '@/utils/ofetch';
import puppeteer from '@/utils/puppeteer';
import { BASE_URL, enhanceItemsWithSummaries, parseArticles } from './utils';
import { BASE_URL, enhanceItemsWithSummaries, fetchPageHtml, parseArticles } from './utils';
export const route: Route = {
path: '/tag/:tag/:sort?',
@ -22,7 +22,7 @@ export const route: Route = {
},
features: {
requireConfig: false,
requirePuppeteer: false,
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
@ -43,17 +43,21 @@ async function handler(ctx) {
const { tag, sort = 'trending' } = ctx.req.param();
const url = `${BASE_URL}/${tag}/${sort}`;
const response = await ofetch(url);
const $ = load(response);
const browser = await puppeteer();
try {
const html = await fetchPageHtml(browser, url, 'article');
const $ = load(html);
const list = parseArticles($, BASE_URL);
const items = await enhanceItemsWithSummaries(browser, list);
const list = parseArticles($, BASE_URL);
const items = await enhanceItemsWithSummaries(list);
const pageTitle = $('title').text() || `DailyPush - ${tag.charAt(0).toUpperCase() + tag.slice(1)}`;
const pageTitle = $('title').text() || `DailyPush - ${tag.charAt(0).toUpperCase() + tag.slice(1)}`;
return {
title: pageTitle,
link: url,
item: items,
};
return {
title: pageTitle,
link: url,
item: items,
};
} finally {
await browser.close();
}
}

59
lib/routes/dailypush/utils.ts
View File

@ -1,9 +1,10 @@
import type { CheerioAPI } from 'cheerio';
import { load } from 'cheerio';
import type { Browser, Page } from 'rebrowser-puppeteer';
import type { DataItem } from '@/types';
import cache from '@/utils/cache';
import ofetch from '@/utils/ofetch';
import logger from '@/utils/logger';
import { parseRelativeDate } from '@/utils/parse-date';
export const BASE_URL = 'https://www.dailypush.dev';
@ -19,6 +20,38 @@ export interface ArticleItem {
dailyPushUrl?: string;
}
const allowedRequestTypes = new Set(['document']);
async function preparePage(page: Page) {
await page.setRequestInterception(true);
page.on('request', (request) => {
if (allowedRequestTypes.has(request.resourceType())) {
request.continue();
return;
}
request.abort();
});
}
export async function fetchPageHtml(browser: Browser, url: string, waitForSelector?: string): Promise<string> {
const page = await browser.newPage();
await preparePage(page);
try {
logger.http(`Requesting ${url}`);
await page.goto(url, { waitUntil: 'domcontentloaded' });
if (waitForSelector) {
await page.waitForSelector(waitForSelector);
}
return await page.content();
} finally {
await page.close();
}
}
/**
* Try to parse text as a date. Returns the Date if parsing succeeds and is valid, undefined otherwise.
*/
@ -40,14 +73,14 @@ function extractAuthor(article: ReturnType<CheerioAPI>): DataItem['author'] {
return undefined;
}
// Get all content spans (exclude separator spans with '•')
// Get all content spans (exclude separator spans with "•")
const allSpans = container.find('span');
const contentSpans: string[] = [];
for (let i = 0; i < allSpans.length; i++) {
const $span = allSpans.eq(i);
const text = $span.text().trim();
// Skip separator spans (contain only '•' or have separator classes)
// Skip separator spans (contain only "•" or have separator classes)
if (text !== '•' && !$span.hasClass('text-slate-300') && !$span.hasClass('dark:text-slate-600')) {
contentSpans.push(text);
}
@ -127,14 +160,14 @@ function extractPubDate(article: ReturnType<CheerioAPI>): Date | undefined {
return undefined;
}
// Get all content spans (exclude separator spans with '•')
// Get all content spans (exclude separator spans with "•")
const allSpans = container.find('span');
const contentSpans: string[] = [];
for (let i = 0; i < allSpans.length; i++) {
const $span = allSpans.eq(i);
const text = $span.text().trim();
// Skip separator spans (contain only '•' or have separator classes)
// Skip separator spans (contain only "•" or have separator classes)
if (text !== '•' && !$span.hasClass('text-slate-300') && !$span.hasClass('dark:text-slate-600')) {
contentSpans.push(text);
}
@ -225,23 +258,20 @@ export function parseArticles($: CheerioAPI, baseUrl: string): ArticleItem[] {
}
/**
* Enhance items with full summaries from dailypush article pages
* Enhance items with full summaries from dailypush article pages.
* Uses the provided browser; opens a new tab per URL (document requests only). Caller must close the browser.
*/
export async function enhanceItemsWithSummaries(items: ArticleItem[]): Promise<DataItem[]> {
export async function enhanceItemsWithSummaries(browser: Browser, items: ArticleItem[]): Promise<DataItem[]> {
const itemsWithUrl = items.filter((item) => item.dailyPushUrl !== undefined);
const itemsWithoutUrl: DataItem[] = items.filter((item) => item.dailyPushUrl === undefined);
const enhancedItems: DataItem[] = await Promise.all(
const enhancedItems = await Promise.all(
itemsWithUrl.map((item) =>
cache.tryGet(item.dailyPushUrl!, async () => {
// If we have a dailypush article URL, fetch it for the longer summary
try {
const articleResponse = await ofetch(item.dailyPushUrl!);
const $ = load(articleResponse);
// Find the longer summary/description on the article page
const html = await fetchPageHtml(browser, item.dailyPushUrl!, 'p.font-ibm-plex-sans.leading-relaxed');
const $ = load(html);
const summary = $('p.font-ibm-plex-sans.leading-relaxed').first();
if (summary.length > 0 && summary.text().trim()) {
item.description = summary.text().trim();
}
@ -254,6 +284,5 @@ export async function enhanceItemsWithSummaries(items: ArticleItem[]): Promise<D
)
);
// Include items without dailyPushUrl as-is
return [...enhancedItems, ...itemsWithoutUrl];
}

4
lib/routes/iwara/ranking.ts
View File

@ -58,7 +58,7 @@ async function handler(ctx) {
const items = await cache.tryGet(
`iwara:ranking:${type}:${sort}:${rating}`,
async () => {
const { page, destory } = await getPuppeteerPage(url, {
const { page, destroy } = await getPuppeteerPage(url, {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
@ -83,7 +83,7 @@ async function handler(ctx) {
pubDate: parseDate(item.createdAt),
}));
} finally {
await destory();
await destroy();
}
},
config.cache.routeExpire,

18
lib/routes/iwara/subscriptions.ts
View File

@ -69,7 +69,7 @@ async function handler() {
const username = config.iwara.username;
const password = config.iwara.password;
const { page, destory } = await getPuppeteerPage(rootUrl, {
const { page, destroy } = await getPuppeteerPage(rootUrl, {
gotoConfig: {
waitUntil: 'domcontentloaded',
},
@ -113,7 +113,10 @@ async function handler() {
async () => {
const result = await fetchApi(`${apiqRootUrl}/user/token`, {
method: 'POST',
headers: { ...apiHeaders, Authorization: refreshHeaders.authorization },
headers: {
...apiHeaders,
Authorization: refreshHeaders.authorization,
},
});
return { authorization: 'Bearer ' + result.accessToken };
},
@ -121,7 +124,10 @@ async function handler() {
false
);
const authedHeaders = { ...apiHeaders, Authorization: authHeaders.authorization };
const authedHeaders = {
...apiHeaders,
Authorization: authHeaders.authorization,
};
// fetch subscriptions
const [videoResponse, imageResponse] = await Promise.all([
@ -177,7 +183,9 @@ async function handler() {
}
const apiUrl = item.link.replace('www.iwara.tv', 'apiq.iwara.tv');
const response = await fetchApi(apiUrl, { headers: authedHeaders });
const response = await fetchApi(apiUrl, {
headers: authedHeaders,
});
description = renderSubscriptionImages(response.files ? response.files.filter((f) => f.type === 'image').map((f) => `${imageRootUrl}/image/original/${f.id}/${f.name}`) : [item.imageUrl]);
@ -202,6 +210,6 @@ async function handler() {
item: items,
};
} finally {
await destory();
await destroy();
}
}

4
lib/routes/nhentai/util.tsx
View File

@ -73,7 +73,7 @@ const fetchPage = async (url: string): Promise<string> => {
} catch (error: unknown) {
const status = (error as { status?: number; statusCode?: number }).status ?? (error as { status?: number; statusCode?: number }).statusCode;
if (status === 403) {
const { page, destory } = await getPuppeteerPage(url, {
const { page, destroy } = await getPuppeteerPage(url, {
onBeforeLoad: async (page) => {
const allowedTypes = new Set(['document', 'script', 'xhr', 'fetch']);
await page.setRequestInterception(true);
@ -83,7 +83,7 @@ const fetchPage = async (url: string): Promise<string> => {
},
});
const content = await page.content();
await destory();
await destroy();
return content;
}
throw error;

8
lib/routes/perplexity/blog.ts
View File

@ -38,7 +38,7 @@ async function handler(ctx: Context) {
const limit = Number.parseInt(ctx.req.query('limit') ?? '20', 10);
const rootUrl = 'https://www.perplexity.ai/hub';
const { page, destory, browser } = await getPuppeteerPage(rootUrl, {
const { page, destroy, browser } = await getPuppeteerPage(rootUrl, {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
@ -119,7 +119,9 @@ async function handler(ctx: Context) {
request.resourceType() === 'document' ? request.continue() : request.abort();
});
await contentPage.goto(item.link!, { waitUntil: 'domcontentloaded' });
await contentPage.goto(item.link!, {
waitUntil: 'domcontentloaded',
});
const contentHtml = await contentPage.evaluate(() => document.documentElement.innerHTML);
await contentPage.close();
@ -148,7 +150,7 @@ async function handler(ctx: Context) {
})
);
await destory();
await destroy();
return {
title: 'Perplexity Blog',

4
lib/routes/perplexity/changelog.ts
View File

@ -16,7 +16,7 @@ export const handler = async (ctx: Context): Promise<Data> => {
logger.http(`Fetching Perplexity changelog from ${targetUrl}`);
const { page, destory, browser } = await getPuppeteerPage(targetUrl, {
const { page, destroy, browser } = await getPuppeteerPage(targetUrl, {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
@ -131,7 +131,7 @@ export const handler = async (ctx: Context): Promise<Data> => {
);
// Close the browser session after all requests are done
await destory();
await destroy();
return {
title: $('title').text() || 'Perplexity Changelog',

4
lib/routes/picnob/utils.ts
View File

@ -2,7 +2,7 @@ import { getPuppeteerPage } from '@/utils/puppeteer';
const puppeteerGet = async (url) => {
let data;
const { destory } = await getPuppeteerPage(url, {
const { destroy } = await getPuppeteerPage(url, {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
@ -13,7 +13,7 @@ const puppeteerGet = async (url) => {
});
},
});
await destory();
await destroy();
return data;
};

4
lib/routes/picuki/profile.ts
View File

@ -85,7 +85,7 @@ async function handler(ctx) {
});
} catch (error) {
if (error.status === 403) {
const { page, destory } = await getPuppeteerPage(profileUrl, {
const { page, destroy } = await getPuppeteerPage(profileUrl, {
onBeforeLoad: async (page) => {
const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']);
await page.setRequestInterception(true);
@ -96,7 +96,7 @@ async function handler(ctx) {
});
await page.waitForSelector('.content');
response = await page.content();
await destory();
await destroy();
} else {
throw new NotFoundError(error.message);
}

4
lib/routes/weibo/utils.ts
View File

@ -80,7 +80,7 @@ const weiboUtils = {
logger.info(`Fetching visitor Cookies from ${url}`);
}
let times = 0;
const { page, destory } = await getPuppeteerPage(url, {
const { page, destroy } = await getPuppeteerPage(url, {
onBeforeLoad: async (page) => {
const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']);
await page.setUserAgent(weiboUtils.apiHeaders['User-Agent']);
@ -101,7 +101,7 @@ const weiboUtils = {
gotoConfig: { waitUntil: 'networkidle0' },
});
const cookies: string = await getCookies(page, 'weibo.cn');
await destory();
await destroy();
if (times < 2 || !cookies) {
throw new Error(`Unable to fetch visitor cookies. Please set WEIBO_COOKIES. Redirection: ${times}, last URL: ${page.url()}`);
}

4
lib/routes/xiaohongshu/util.ts
View File

@ -63,7 +63,7 @@ const getUser = (url, cache) =>
}
// Use puppeteer
const { page, destory } = await getPuppeteerPage(url, {
const { page, destroy } = await getPuppeteerPage(url, {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
@ -107,7 +107,7 @@ const getUser = (url, cache) =>
return { userPageData, notes, collect };
} finally {
await destory();
await destroy();
}
},
config.cache.routeExpire,

2
lib/utils/puppeteer.mock.test.ts
View File

@ -69,7 +69,7 @@ describe('getPuppeteerPage (mocked)', () => {
expect(endpoint).toContain('stealth=true');
expect(onBeforeLoad).toHaveBeenCalled();
await result.destory();
await result.destroy();
expect(browser.close).toHaveBeenCalled();
delete process.env.PUPPETEER_WS_ENDPOINT;

2
lib/utils/puppeteer.ts
View File

@ -187,7 +187,7 @@ export const getPuppeteerPage = async (
return {
page,
destory: async () => {
destroy: async () => {
await browser.close();
},
browser,

2
lib/utils/puppeteer.worker.ts
View File

@ -91,7 +91,7 @@ export const getPuppeteerPage = async (
return {
page,
destory: async () => {
destroy: async () => {
await browser.close();
},
browser,