From df320988d0482f19d4e289e1d97a3ea6e0c3c2bf Mon Sep 17 00:00:00 2001 From: Tony Date: Wed, 23 Apr 2025 21:02:10 +0800 Subject: [PATCH] refactor: replace tiny-async-pool with p-map (#18928) * refactor: replace tiny-async-pool with p-map * fix: use original concurrency --- lib/routes/agefans/update.ts | 40 +++---- lib/routes/apnews/mobile-api.ts | 5 +- lib/routes/apnews/sitemap.ts | 5 +- lib/routes/apnews/topics.ts | 5 +- lib/routes/apnews/utils.ts | 8 -- lib/routes/bdys/index.ts | 65 ++++++----- lib/routes/bjnews/cat.ts | 11 +- lib/routes/bjx/huanbao.ts | 16 +-- lib/routes/bloomberg/authors.ts | 5 +- lib/routes/bloomberg/index.ts | 11 +- lib/routes/bloomberg/utils.ts | 11 +- lib/routes/cara/utils.ts | 9 -- lib/routes/cfr/index.ts | 5 +- lib/routes/cfr/utils.ts | 9 -- lib/routes/copymanga/comic.ts | 12 +- lib/routes/cpta/handler.ts | 8 +- lib/routes/cts/news.ts | 45 ++++---- lib/routes/dcard/utils.ts | 75 ++++++------ lib/routes/dlnews/category.ts | 7 +- lib/routes/gdut/oa-news.ts | 169 ++++++++++++++-------------- lib/routes/gov/nrta/dsj.ts | 27 +++-- lib/routes/guozaoke/index.ts | 69 ++++++------ lib/routes/kcna/news.ts | 64 +++++------ lib/routes/luogu/contest.ts | 50 ++++---- lib/routes/nextapple/realtime.ts | 55 ++++----- lib/routes/shoppingdesign/posts.ts | 51 +++++---- lib/routes/tfc-taiwan/utils.ts | 45 ++++---- lib/routes/tradingview/blog.ts | 80 ++++++------- lib/routes/wsj/news.ts | 5 +- lib/routes/wsj/utils.ts | 10 +- lib/routes/x-mol/paper.ts | 39 +++---- lib/routes/yamibo/bbs/forum.ts | 9 +- lib/routes/yamibo/utils.ts | 9 -- lib/routes/youtube/subscriptions.ts | 14 +-- lib/routes/zaker/channel.ts | 7 +- lib/routes/zaker/focus.ts | 7 +- package.json | 2 - pnpm-lock.yaml | 16 --- 38 files changed, 483 insertions(+), 597 deletions(-) diff --git a/lib/routes/agefans/update.ts b/lib/routes/agefans/update.ts index 37a853d14..8c987e0fd 100644 --- a/lib/routes/agefans/update.ts +++ b/lib/routes/agefans/update.ts @@ -1,9 +1,9 @@ -import { Route } from '@/types'; +import { DataItem, Route } from '@/types'; import cache from '@/utils/cache'; import got from '@/utils/got'; import { load } from 'cheerio'; import { rootUrl } from './utils'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; export const route: Route = { path: '/update', @@ -47,27 +47,27 @@ async function handler() { }; }); - const items: any[] = []; - for await (const item of asyncPool(3, list, (item) => - cache.tryGet(item.link, async () => { - const detailResponse = await got(item.link); - const content = load(detailResponse.data); + const items: DataItem[] = await pMap( + list, + (item) => + cache.tryGet(item.link, async () => { + const detailResponse = await got(item.link); + const content = load(detailResponse.data); - content('img').each((_, ele) => { - if (ele.attribs['data-original']) { - ele.attribs.src = ele.attribs['data-original']; - delete ele.attribs['data-original']; - } - }); - content('.video_detail_collect').remove(); + content('img').each((_, ele) => { + if (ele.attribs['data-original']) { + ele.attribs.src = ele.attribs['data-original']; + delete ele.attribs['data-original']; + } + }); + content('.video_detail_collect').remove(); - item.description = content('.video_detail_left').html(); + item.description = content('.video_detail_left').html(); - return item; - }) - )) { - items.push(item); - } + return item; + }), + { concurrency: 3 } + ); return { title: $('title').text(), diff --git a/lib/routes/apnews/mobile-api.ts b/lib/routes/apnews/mobile-api.ts index 078688595..34cce0f0c 100644 --- a/lib/routes/apnews/mobile-api.ts +++ b/lib/routes/apnews/mobile-api.ts @@ -1,5 +1,6 @@ import { Route, ViewType } from '@/types'; -import { asyncPoolAll, fetchArticle } from './utils'; +import { fetchArticle } from './utils'; +import pMap from 'p-map'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; @@ -83,7 +84,7 @@ async function handler(ctx) { .sort((a, b) => b.pubDate - a.pubDate) .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20); - const items = ctx.req.query('fulltext') === 'true' ? await asyncPoolAll(10, list, (item) => fetchArticle(item)) : list; + const items = ctx.req.query('fulltext') === 'true' ? await pMap(list, (item) => fetchArticle(item), { concurrency: 10 }) : list; return { title: screen.category ?? screen.title, diff --git a/lib/routes/apnews/sitemap.ts b/lib/routes/apnews/sitemap.ts index 655ab7f08..272d12ff0 100644 --- a/lib/routes/apnews/sitemap.ts +++ b/lib/routes/apnews/sitemap.ts @@ -1,5 +1,6 @@ import { Route, ViewType } from '@/types'; -import { asyncPoolAll, fetchArticle } from './utils'; +import { fetchArticle } from './utils'; +import pMap from 'p-map'; import ofetch from '@/utils/ofetch'; import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; @@ -81,7 +82,7 @@ async function handler(ctx) { .sort((a, b) => (a.pubDate && b.pubDate ? b.pubDate - a.pubDate : b.lastmod - a.lastmod)) .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20); - const items = ctx.req.query('fulltext') === 'true' ? await asyncPoolAll(20, list, (item) => fetchArticle(item)) : list; + const items = ctx.req.query('fulltext') === 'true' ? await pMap(list, (item) => fetchArticle(item), { concurrency: 20 }) : list; return { title: `AP News sitemap:${route}`, diff --git a/lib/routes/apnews/topics.ts b/lib/routes/apnews/topics.ts index a4f65f319..96185f087 100644 --- a/lib/routes/apnews/topics.ts +++ b/lib/routes/apnews/topics.ts @@ -1,7 +1,8 @@ import { Route, ViewType } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { asyncPoolAll, fetchArticle, removeDuplicateByKey } from './utils'; +import { fetchArticle, removeDuplicateByKey } from './utils'; +import pMap from 'p-map'; const HOME_PAGE = 'https://apnews.com'; export const route: Route = { @@ -50,7 +51,7 @@ async function handler(ctx) { })) .filter((e) => typeof e.link === 'string'); - const items = ctx.req.query('fulltext') === 'true' ? await asyncPoolAll(10, list, (item) => fetchArticle(item)) : list; + const items = ctx.req.query('fulltext') === 'true' ? await pMap(list, (item) => fetchArticle(item), { concurrency: 10 }) : list; return { title: $('title').text(), diff --git a/lib/routes/apnews/utils.ts b/lib/routes/apnews/utils.ts index ed833750c..02b462f62 100644 --- a/lib/routes/apnews/utils.ts +++ b/lib/routes/apnews/utils.ts @@ -2,7 +2,6 @@ import cache from '@/utils/cache'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; import { load } from 'cheerio'; -import asyncPool from 'tiny-async-pool'; export function removeDuplicateByKey(items, key: string) { return [...new Map(items.map((x) => [x[key], x])).values()]; @@ -65,10 +64,3 @@ export function fetchArticle(item) { } }); } -export async function asyncPoolAll(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise) { - const results: Awaited = []; - for await (const result of asyncPool(poolLimit, array, iteratorFn)) { - results.push(result); - } - return results; -} diff --git a/lib/routes/bdys/index.ts b/lib/routes/bdys/index.ts index b3082c3fc..2a0c7527a 100644 --- a/lib/routes/bdys/index.ts +++ b/lib/routes/bdys/index.ts @@ -7,7 +7,7 @@ import { parseDate } from '@/utils/parse-date'; import timezone from '@/utils/timezone'; import { art } from '@/utils/render'; import path from 'node:path'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; import { config } from '@/config'; import ConfigNotFoundError from '@/errors/types/config-not-found'; @@ -139,43 +139,42 @@ async function handler(ctx) { cookie: `JSESSIONID=${jsessionid}`, }; - const items = []; + const items = await pMap( + list, + (item) => + cache.tryGet(item.link, async () => { + const detailResponse = await got({ + method: 'get', + url: item.link, + headers, + }); + const downloadResponse = await got({ + method: 'get', + url: `${rootUrl}/downloadInfo/list?mid=${item.link.split('/')[4].split('.')[0]}`, + headers, + }); + const content = load(detailResponse.data); - for await (const data of asyncPool(1, list, (item) => - cache.tryGet(item.link, async () => { - const detailResponse = await got({ - method: 'get', - url: item.link, - headers, - }); - const downloadResponse = await got({ - method: 'get', - url: `${rootUrl}/downloadInfo/list?mid=${item.link.split('/')[4].split('.')[0]}`, - headers, - }); - const content = load(detailResponse.data); + content('svg').remove(); + const torrents = content('.download-list .list-group'); - content('svg').remove(); - const torrents = content('.download-list .list-group'); + item.description = art(path.join(__dirname, 'templates/desc.art'), { + info: content('.row.mt-3').html(), + synopsis: content('#synopsis').html(), + links: downloadResponse.data, + torrents: torrents.html(), + }); - item.description = art(path.join(__dirname, 'templates/desc.art'), { - info: content('.row.mt-3').html(), - synopsis: content('#synopsis').html(), - links: downloadResponse.data, - torrents: torrents.html(), - }); + item.pubDate = timezone(parseDate(content('.bg-purple-lt').text().replace('更新时间：', '')), +8); + item.guid = `${item.link}#${content('.card h1').text()}`; - item.pubDate = timezone(parseDate(content('.bg-purple-lt').text().replace('更新时间：', '')), +8); - item.guid = `${item.link}#${content('.card h1').text()}`; + item.enclosure_url = torrents.html() ? `${rootUrl}${torrents.find('a').first().attr('href')}` : downloadResponse.data.pop().url; + item.enclosure_type = 'application/x-bittorrent'; - item.enclosure_url = torrents.html() ? `${rootUrl}${torrents.find('a').first().attr('href')}` : downloadResponse.data.pop().url; - item.enclosure_type = 'application/x-bittorrent'; - - return item; - }) - )) { - items.push(data); - } + return item; + }), + { concurrency: 1 } + ); return { title: '哔嘀影视', diff --git a/lib/routes/bjnews/cat.ts b/lib/routes/bjnews/cat.ts index 4ce068e8f..c88801b3e 100644 --- a/lib/routes/bjnews/cat.ts +++ b/lib/routes/bjnews/cat.ts @@ -3,7 +3,7 @@ import { load } from 'cheerio'; import ofetch from '@/utils/ofetch'; import { fetchArticle } from './utils'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; export const route: Route = { path: '/cat/:cat', @@ -35,17 +35,10 @@ async function handler(ctx) { category: $(a).parent().find('.source').text().trim(), })); - const out = await asyncPoolAll(2, list, (item) => fetchArticle(item)); + const out = await pMap(list, (item) => fetchArticle(item), { concurrency: 2 }); return { title: `新京报 - 分类 - ${$('.cur').text().trim()}`, link: url, item: out, }; } -async function asyncPoolAll(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise) { - const results: Awaited = []; - for await (const result of asyncPool(poolLimit, array, iteratorFn)) { - results.push(result); - } - return results; -} diff --git a/lib/routes/bjx/huanbao.ts b/lib/routes/bjx/huanbao.ts index 5d53114e1..302471c8d 100644 --- a/lib/routes/bjx/huanbao.ts +++ b/lib/routes/bjx/huanbao.ts @@ -4,15 +4,7 @@ import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; import { load } from 'cheerio'; import timezone from '@/utils/timezone'; -import asyncPool from 'tiny-async-pool'; - -const asyncPoolAll = async (...args) => { - const results = []; - for await (const result of asyncPool(...args)) { - results.push(result); - } - return results; -}; +import pMap from 'p-map'; export const route: Route = { path: '/huanbao', @@ -54,11 +46,11 @@ async function handler() { }; }); - items = await asyncPoolAll( + items = await pMap( // 服务器禁止单个IP大并发访问，只能少返回几条 - 3, items, - (items) => fetchPage(items.link) + (item) => fetchPage(item.link), + { concurrency: 3 } ); return { diff --git a/lib/routes/bloomberg/authors.ts b/lib/routes/bloomberg/authors.ts index 83b6eed85..8678b6f7b 100644 --- a/lib/routes/bloomberg/authors.ts +++ b/lib/routes/bloomberg/authors.ts @@ -2,7 +2,8 @@ import { Route, ViewType } from '@/types'; import { load } from 'cheerio'; import ofetch from '@/utils/ofetch'; import rssParser from '@/utils/rss-parser'; -import { asyncPoolAll, parseArticle } from './utils'; +import { parseArticle } from './utils'; +import pMap from 'p-map'; const parseAuthorNewsList = async (slug) => { const baseURL = `https://www.bloomberg.com/authors/${slug}`; @@ -66,7 +67,7 @@ async function handler(ctx) { list = (await rssParser.parseURL(`${link}.rss`)).items; } - const item = await asyncPoolAll(1, list, (item) => parseArticle(item)); + const item = await pMap(list, (item) => parseArticle(item), { concurrency: 1 }); const authorName = item.find((i) => i.author)?.author ?? slug; return { diff --git a/lib/routes/bloomberg/index.ts b/lib/routes/bloomberg/index.ts index 46f6a7d8d..bedc603c0 100644 --- a/lib/routes/bloomberg/index.ts +++ b/lib/routes/bloomberg/index.ts @@ -1,6 +1,7 @@ import { Route, ViewType } from '@/types'; -import { rootUrl, asyncPoolAll, parseNewsList, parseArticle } from './utils'; -const site_title_mapping = { +import { rootUrl, parseNewsList, parseArticle } from './utils'; +import pMap from 'p-map'; +const siteTitleMapping = { '/': 'News', bpol: 'Politics', bbiz: 'Business', @@ -23,7 +24,7 @@ export const route: Route = { parameters: { site: { description: 'Site ID, can be found below', - options: Object.keys(site_title_mapping).map((key) => ({ value: key, label: site_title_mapping[key] })), + options: Object.keys(siteTitleMapping).map((key) => ({ value: key, label: siteTitleMapping[key] })), }, }, features: { @@ -60,9 +61,9 @@ async function handler(ctx) { const currentUrl = site ? `${rootUrl}/${site}/sitemap_news.xml` : `${rootUrl}/sitemap_news.xml`; const list = await parseNewsList(currentUrl, ctx); - const items = await asyncPoolAll(1, list, (item) => parseArticle(item)); + const items = await pMap(list, (item) => parseArticle(item), { concurrency: 1 }); return { - title: `Bloomberg - ${site_title_mapping[site ?? '/']}`, + title: `Bloomberg - ${siteTitleMapping[site ?? '/']}`, link: currentUrl, item: items, }; diff --git a/lib/routes/bloomberg/utils.ts b/lib/routes/bloomberg/utils.ts index e9d25a07b..7736a6703 100644 --- a/lib/routes/bloomberg/utils.ts +++ b/lib/routes/bloomberg/utils.ts @@ -1,7 +1,6 @@ import cache from '@/utils/cache'; import { load } from 'cheerio'; import path from 'node:path'; -import asyncPool from 'tiny-async-pool'; import { destr } from 'destr'; import { parseDate } from '@/utils/parse-date'; @@ -604,12 +603,4 @@ const documentToHtmlString = async (document) => { return str; }; -const asyncPoolAll = async (...args) => { - const results = []; - for await (const result of asyncPool(...args)) { - results.push(result); - } - return results; -}; - -export { rootUrl, asyncPoolAll, parseNewsList, parseArticle }; +export { rootUrl, parseNewsList, parseArticle }; diff --git a/lib/routes/cara/utils.ts b/lib/routes/cara/utils.ts index a68bb5f87..92d1ce4c4 100644 --- a/lib/routes/cara/utils.ts +++ b/lib/routes/cara/utils.ts @@ -1,7 +1,6 @@ import { config } from '@/config'; import ofetch from '@/utils/ofetch'; import type { FetchOptions, FetchRequest, ResponseType } from 'ofetch'; -import asyncPool from 'tiny-async-pool'; import type { PortfolioDetailResponse, PortfolioResponse, UserNextData } from './types'; import type { DataItem } from '@/types'; import { parseDate } from '@/utils/parse-date'; @@ -35,14 +34,6 @@ export async function parseUserData(user: string) { })) as Promise; } -export async function asyncPoolAll(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise) { - const results: Awaited = []; - for await (const result of asyncPool(poolLimit, array, iteratorFn)) { - results.push(result); - } - return results; -} - export async function fetchPortfolioItem(item: PortfolioResponse['data'][number]) { const res = await customFetch(`${API_HOST}/posts/${item.postId}`); diff --git a/lib/routes/cfr/index.ts b/lib/routes/cfr/index.ts index a1d128de4..efcd50804 100644 --- a/lib/routes/cfr/index.ts +++ b/lib/routes/cfr/index.ts @@ -2,7 +2,8 @@ import type { Data, Route } from '@/types'; import type { Context } from 'hono'; import ofetch from '@/utils/ofetch'; import { load } from 'cheerio'; -import { asyncPoolAll, getDataItem } from './utils'; +import { getDataItem } from './utils'; +import pMap from 'p-map'; export const route: Route = { path: '/:category/:subCategory?', @@ -48,7 +49,7 @@ async function handler(ctx: Context): Promise { const listSelector = selectorMap[category] ?? '.card-article-large__link'; - const items = await asyncPoolAll(5, $(listSelector).toArray(), async (item) => await getDataItem($(item).attr('href')!)); + const items = await pMap($(listSelector).toArray(), (item) => getDataItem($(item).attr('href')!), { concurrency: 5 }); return { title: $('head title').text().replace(' | Council on Foreign Relations', ''), diff --git a/lib/routes/cfr/utils.ts b/lib/routes/cfr/utils.ts index 7205f690b..753994413 100644 --- a/lib/routes/cfr/utils.ts +++ b/lib/routes/cfr/utils.ts @@ -4,7 +4,6 @@ import type { DataItem } from '@/types'; import { parseDate } from '@/utils/parse-date'; import cache from '@/utils/cache'; import type { LinkData, VideoSetup } from './types'; -import asyncPool from 'tiny-async-pool'; export function getDataItem(href: string) { const origin = 'https://www.cfr.org'; @@ -274,11 +273,3 @@ function parseDescription($description: Cheerio, $: CheerioAPI) { return description; } - -export async function asyncPoolAll(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise) { - const results: Awaited = []; - for await (const result of asyncPool(poolLimit, array, iteratorFn)) { - results.push(result); - } - return results; -} diff --git a/lib/routes/copymanga/comic.ts b/lib/routes/copymanga/comic.ts index 58176816d..be2093b7c 100644 --- a/lib/routes/copymanga/comic.ts +++ b/lib/routes/copymanga/comic.ts @@ -7,7 +7,7 @@ import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; import { config } from '@/config'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; export const route: Route = { path: '/comic/:id/:chapterCnt?', @@ -126,15 +126,7 @@ async function handler(ctx) { }; }; - const asyncPoolAll = async (...args) => { - const results = []; - for await (const result of asyncPool(...args)) { - results.push(result); - } - return results; - }; - - const result = await asyncPoolAll(3, chapterArray.slice(0, chapterCnt), (chapter) => cache.tryGet(chapter.link, () => genResult(chapter))); + const result = await pMap(chapterArray.slice(0, chapterCnt), (chapter) => cache.tryGet(chapter.link, () => genResult(chapter)), { concurrency: 3 }); const items = [...result, ...chapterArray.slice(chapterCnt)]; return { diff --git a/lib/routes/cpta/handler.ts b/lib/routes/cpta/handler.ts index d5f34af89..7902e5e23 100644 --- a/lib/routes/cpta/handler.ts +++ b/lib/routes/cpta/handler.ts @@ -2,7 +2,7 @@ import { DataItem, Route } from '@/types'; import cache from '@/utils/cache'; import got from '@/utils/got'; import { load } from 'cheerio'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; type NewsCategory = { title: string; @@ -77,11 +77,7 @@ const handler: Route['handler'] = async (ctx) => { } as DataItem; }); - const dataItems: DataItem[] = []; - - for await (const item of await asyncPool(1, contentLinkList, fetchDataItem)) { - dataItems.push(item as DataItem); - } + const dataItems: DataItem[] = await pMap(contentLinkList, fetchDataItem, { concurrency: 1 }); return { title: `中国人事考试网-${NEWS_TYPES[category].title}`, diff --git a/lib/routes/cts/news.ts b/lib/routes/cts/news.ts index c06f49020..2e9c46511 100644 --- a/lib/routes/cts/news.ts +++ b/lib/routes/cts/news.ts @@ -3,7 +3,7 @@ import cache from '@/utils/cache'; import got from '@/utils/got'; import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; export const route: Route = { path: '/:category', @@ -36,28 +36,29 @@ async function handler(ctx) { const currentUrl = `https://news.cts.com.tw/${category}/index.html`; const response = await got(currentUrl); const $ = load(response.data); - const items = []; - for await (const data of asyncPool(5, $('#newslist-top a[title]').slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 20), (item) => { - item = $(item); - const link = item.attr('href'); - return cache.tryGet(link, async () => { - const response = await got(link); - const $ = load(response.data); - const author = $('.artical-content p:eq(0)').text().trim(); - $('.artical-content p:eq(0), .artical-content .flexbox').remove(); + const items = await pMap( + $('#newslist-top a[title]').slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 20), + (item) => { + item = $(item); + const link = item.attr('href'); + return cache.tryGet(link, async () => { + const response = await got(link); + const $ = load(response.data); + const author = $('.artical-content p:eq(0)').text().trim(); + $('.artical-content p:eq(0), .artical-content .flexbox').remove(); - return { - title: item.attr('title'), - author, - description: $('.artical-content').html(), - category: $('meta[property="article:section"]').attr('content'), - pubDate: parseDate($('meta[property="article:published_time"]').attr('content')), - link, - }; - }); - })) { - items.push(data); - } + return { + title: item.attr('title'), + author, + description: $('.artical-content').html(), + category: $('meta[property="article:section"]').attr('content'), + pubDate: parseDate($('meta[property="article:published_time"]').attr('content')), + link, + }; + }); + }, + { concurrency: 5 } + ); return { title: $('title').text(), diff --git a/lib/routes/dcard/utils.ts b/lib/routes/dcard/utils.ts index 3db10ad40..083c736d6 100644 --- a/lib/routes/dcard/utils.ts +++ b/lib/routes/dcard/utils.ts @@ -1,46 +1,47 @@ -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; const ProcessFeed = async (items, cookies, browser, limit, cache) => { let newCookies = []; - const result = []; - for await (const item of asyncPool(3, items.slice(0, limit), async (i) => { - const url = `https://www.dcard.tw/service/api/v2/posts/${i.id}`; - const content = await cache.tryGet(`dcard:${i.id}`, async () => { - let response; - // try catch 处理被删除的帖子 - try { - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? request.continue() : request.abort(); - }); - await page.setExtraHTTPHeaders({ - referer: `https://www.dcard.tw/f/${i.forumAlias}/p/${i.id}`, - }); - await page.setCookie(...cookies); - await page.goto(url); - await page.waitForSelector('body > pre'); - response = await page.evaluate(() => document.querySelector('body > pre').textContent); - newCookies = await page.cookies(); - await page.close(); + const result = await pMap( + items.slice(0, limit), + async (i) => { + const url = `https://www.dcard.tw/service/api/v2/posts/${i.id}`; + const content = await cache.tryGet(`dcard:${i.id}`, async () => { + let response; + // try catch 处理被删除的帖子 + try { + const page = await browser.newPage(); + await page.setRequestInterception(true); + page.on('request', (request) => { + request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? request.continue() : request.abort(); + }); + await page.setExtraHTTPHeaders({ + referer: `https://www.dcard.tw/f/${i.forumAlias}/p/${i.id}`, + }); + await page.setCookie(...cookies); + await page.goto(url); + await page.waitForSelector('body > pre'); + response = await page.evaluate(() => document.querySelector('body > pre').textContent); + newCookies = await page.cookies(); + await page.close(); - const data = JSON.parse(response); - let body = data.content; - body = body.replaceAll(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => ``); - body = body.replaceAll(/(?=https?:\/\/).*(??)$/gim, (m) => `${m}`); - body = body.replaceAll('\n', '
'); + const data = JSON.parse(response); + let body = data.content; + body = body.replaceAll(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => ``); + body = body.replaceAll(/(?=https?:\/\/).*(??)$/gim, (m) => `${m}`); + body = body.replaceAll('\n', '
'); - return body; - } catch { - return ''; - } - }); + return body; + } catch { + return ''; + } + }); - i.description = content; - return i; - })) { - result.push(item); - } + i.description = content; + return i; + }, + { concurrency: 3 } + ); await cache.set('dcard:cookies', newCookies, 3600); return [...result, ...items.slice(limit)]; }; diff --git a/lib/routes/dlnews/category.ts b/lib/routes/dlnews/category.ts index 156db563f..3ec34bedc 100644 --- a/lib/routes/dlnews/category.ts +++ b/lib/routes/dlnews/category.ts @@ -6,7 +6,7 @@ import got from '@/utils/got'; import { getData, getList } from './utils'; import { art } from '@/utils/render'; import path from 'node:path'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; const _website = 'dlnews'; const topics = { @@ -91,10 +91,7 @@ async function handler(ctx) { }; const data = await getData(`${baseUrl}${apiPath}?query=${encodeURIComponent(JSON.stringify(query))}&_website=${_website}`); const list = getList(data); - const items = []; - for await (const data of asyncPool(3, list, (item) => extractArticle(item))) { - items.push(data); - } + const items = await pMap(list, (item) => extractArticle(item), { concurrency: 3 }); return { title: Object.hasOwn(topics, category) ? `${topics[category]} : DL News` : 'DL News', diff --git a/lib/routes/gdut/oa-news.ts b/lib/routes/gdut/oa-news.ts index 58c8128b8..60d5c3c10 100644 --- a/lib/routes/gdut/oa-news.ts +++ b/lib/routes/gdut/oa-news.ts @@ -5,7 +5,7 @@ import { load } from 'cheerio'; import { CookieJar } from 'tough-cookie'; import { parseDate } from '@/utils/parse-date'; import timezone from '@/utils/timezone'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; const site = 'https://oas.gdut.edu.cn/seeyon'; const typeMap = { @@ -123,94 +123,95 @@ async function handler(ctx) { category: item.typeName, })); - const results = []; - // 获取实际的文章内容 - for await (const data of asyncPool(2, articles, async (data) => { - const link = data.link; - data.description = await cache.tryGet(link, async () => { - // 获取数据 - const response = await got(link, { - cookieJar, - }); + const results = await pMap( + articles, + async (data) => { + const link = data.link; + data.description = await cache.tryGet(link, async () => { + // 获取数据 + const response = await got(link, { + cookieJar, + }); - const $ = load(response.data); - const node = $('#content'); - // 清理样式 - node.find('*') - .filter(function () { - return this.type === 'comment' || this.tagName === 'meta' || this.tagName === 'style'; - }) - .remove(); - node.find('*') - .contents() - .filter(function () { - return this.type === 'comment' || this.tagName === 'meta' || this.tagName === 'style'; - }) - .remove(); - node.find('*').each(function () { - if (this.attribs.style !== undefined) { - const newSty = this.attribs.style - .split(';') - .filter((s) => { - const styBlocklist = ['color:rgb(0,0,0)', 'color:black', 'background:rgb(255,255,255)', 'background:white', 'text-align:left', 'text-align:justify', 'font-style:normal', 'font-weight:normal']; - const styPrefixBlocklist = [ - 'font-family', - 'font-size', - 'background', - 'text-autospace', - 'text-transform', - 'letter-spacing', - 'line-height', - 'padding', - 'margin', - 'text-justify', - 'word-break', - 'vertical-align', - 'mso-', - '-ms-', - ]; - const sty = s.trim(); - if (styBlocklist.includes(sty.replaceAll(/\s+/g, ''))) { - return false; - } - for (const prefix of styPrefixBlocklist) { - if (sty.startsWith(prefix)) { + const $ = load(response.data); + const node = $('#content'); + // 清理样式 + node.find('*') + .filter(function () { + return this.type === 'comment' || this.tagName === 'meta' || this.tagName === 'style'; + }) + .remove(); + node.find('*') + .contents() + .filter(function () { + return this.type === 'comment' || this.tagName === 'meta' || this.tagName === 'style'; + }) + .remove(); + node.find('*').each(function () { + if (this.attribs.style !== undefined) { + const newSty = this.attribs.style + .split(';') + .filter((s) => { + const styBlocklist = ['color:rgb(0,0,0)', 'color:black', 'background:rgb(255,255,255)', 'background:white', 'text-align:left', 'text-align:justify', 'font-style:normal', 'font-weight:normal']; + const styPrefixBlocklist = [ + 'font-family', + 'font-size', + 'background', + 'text-autospace', + 'text-transform', + 'letter-spacing', + 'line-height', + 'padding', + 'margin', + 'text-justify', + 'word-break', + 'vertical-align', + 'mso-', + '-ms-', + ]; + const sty = s.trim(); + if (styBlocklist.includes(sty.replaceAll(/\s+/g, ''))) { return false; } - } - return true; - }) - .join(';'); - if (newSty) { - this.attribs.style = newSty; - } else { - delete this.attribs.style; + for (const prefix of styPrefixBlocklist) { + if (sty.startsWith(prefix)) { + return false; + } + } + return true; + }) + .join(';'); + if (newSty) { + this.attribs.style = newSty; + } else { + delete this.attribs.style; + } } - } - if (this.attribs.class && this.attribs.class.trim().startsWith('Mso')) { - delete this.attribs.class; - } - if (this.attribs.lang) { - delete this.attribs.lang; - } - if (this.tagName === 'font' || this.tagName === 'o:p') { - $(this).replaceWith(this.childNodes); - } - if (this.tagName === 'span' && !this.attribs.style) { - $(this).replaceWith(this.childNodes); - } - }); - node.find('span').each(function () { - if (this.childNodes.length === 0) { - $(this).remove(); - } - }); + if (this.attribs.class && this.attribs.class.trim().startsWith('Mso')) { + delete this.attribs.class; + } + if (this.attribs.lang) { + delete this.attribs.lang; + } + if (this.tagName === 'font' || this.tagName === 'o:p') { + $(this).replaceWith(this.childNodes); + } + if (this.tagName === 'span' && !this.attribs.style) { + $(this).replaceWith(this.childNodes); + } + }); + node.find('span').each(function () { + if (this.childNodes.length === 0) { + $(this).remove(); + } + }); - return node.html(); - }); - })) { - results.push(data); - } + return node.html(); + }); + return data; + }, + { concurrency: 2 } + ); return { title: `广东工业大学新闻通知网 - ` + type.name, diff --git a/lib/routes/gov/nrta/dsj.ts b/lib/routes/gov/nrta/dsj.ts index 39ba4c4f0..275af3afc 100644 --- a/lib/routes/gov/nrta/dsj.ts +++ b/lib/routes/gov/nrta/dsj.ts @@ -3,7 +3,7 @@ import cache from '@/utils/cache'; import got from '@/utils/got'; import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; export const route: Route = { path: '/nrta/dsj/:category?', @@ -52,23 +52,22 @@ async function handler(ctx) { }; }); - const results = []; + const results = await pMap( + items, + (item) => + cache.tryGet(item.link, async () => { + const { data: detailResponse } = await got(item.link); - for await (const item of asyncPool(5, items, (item) => - cache.tryGet(item.link, async () => { - const { data: detailResponse } = await got(item.link); + const content = load(detailResponse); - const content = load(detailResponse); + content('table').last().remove(); - content('table').last().remove(); + item.description = content('td.newstext').html() || content('table').last().parent().parent().html(); - item.description = content('td.newstext').html() || content('table').last().parent().parent().html(); - - return item; - }) - )) { - results.push(item); - } + return item; + }), + { concurrency: 5 } + ); return { item: results, diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index d8fe2c35e..0e71195b4 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -4,7 +4,7 @@ import { load } from 'cheerio'; import { parseRelativeDate } from '@/utils/parse-date'; import { config } from '@/config'; import cache from '@/utils/cache'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; export const route: Route = { path: '/default', @@ -54,42 +54,41 @@ async function handler() { }) .filter((item) => item !== undefined); - const out = []; - for await (const result of asyncPool(2, items, (item) => - cache.tryGet(item.link, async () => { - const url = `https://www.guozaoke.com${item.link}`; - const res = await got({ - method: 'get', - url, - headers: { - Cookie: config.guozaoke.cookies, - 'User-Agent': config.ua, - }, - }); + const out = await pMap( + items, + (item) => + cache.tryGet(item.link, async () => { + const url = `https://www.guozaoke.com${item.link}`; + const res = await got({ + method: 'get', + url, + headers: { + Cookie: config.guozaoke.cookies, + }, + }); - const $ = load(res.data); - let content = $('div.ui-content').html(); - content = content ? content.trim() : ''; - const comments = $('.reply-item').map((i, el) => { - const $el = $(el); - const comment = $el.find('span.content').text().trim(); - const author = $el.find('span.username').text(); - return { - comment, - author, - }; - }); - if (comments && comments.length > 0) { - for (const item of comments) { - content += '
' + item.author + ': ' + item.comment; + const $ = load(res.data); + let content = $('div.ui-content').html(); + content = content ? content.trim() : ''; + const comments = $('.reply-item').map((i, el) => { + const $el = $(el); + const comment = $el.find('span.content').text().trim(); + const author = $el.find('span.username').text(); + return { + comment, + author, + }; + }); + if (comments && comments.length > 0) { + for (const item of comments) { + content += '
' + item.author + ': ' + item.comment; + } } - } - item.description = content; - return item; - }) - )) { - out.push(result); - } + item.description = content; + return item; + }), + { concurrency: 2 } + ); return { title: '过早客', diff --git a/lib/routes/kcna/news.ts b/lib/routes/kcna/news.ts index 7eac72a2e..4239d84db 100644 --- a/lib/routes/kcna/news.ts +++ b/lib/routes/kcna/news.ts @@ -3,7 +3,7 @@ import { Route } from '@/types'; import cache from '@/utils/cache'; import got from '@/utils/got'; import { load } from 'cheerio'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; import { art } from '@/utils/render'; import { fixDesc, fetchPhoto, fetchVideo } from './utils'; import path from 'node:path'; @@ -79,42 +79,42 @@ async function handler(ctx) { // avoid being IP-banned // if being banned, 103.35.255.254 (the last hop before www.kcna.kp - 175.45.176.71) will drop the packet // verify that with `mtr www.kcna.kp -Tz` - const items = []; - for await (const item of asyncPool(3, list, (item) => - cache.tryGet(item.link, async () => { - const response = await got(item.link); - const $ = load(response.data); - item.title = $('article-main-title').text() || item.title; + const items = await pMap( + list, + (item) => + cache.tryGet(item.link, async () => { + const response = await got(item.link); + const $ = load(response.data); + item.title = $('article-main-title').text() || item.title; - const dateElem = $('.publish-time'); - const dateString = dateElem.text().match(/\d+\.\d+\.\d+/); - dateElem.remove(); - item.pubDate = dateString ? timezone(parseDate(dateString[0]), +9) : item.pubDate; + const dateElem = $('.publish-time'); + const dateString = dateElem.text().match(/\d+\.\d+\.\d+/); + dateElem.remove(); + item.pubDate = dateString ? timezone(parseDate(dateString[0]), +9) : item.pubDate; - const description = fixDesc($, $('.article-content-body .content-wrapper')); + const description = fixDesc($, $('.article-content-body .content-wrapper')); - // add picture and video - const media = $('.media-icon a') - .map((_, elem) => rootUrl + elem.attribs.href) - .get(); - let photo, video; - await Promise.all( - media.map(async (medium) => { - if (medium.includes('/photo/')) { - photo = await fetchPhoto(ctx, medium); - } else if (medium.includes('/video/')) { - video = await fetchVideo(ctx, medium); - } - }) - ); + // add picture and video + const media = $('.media-icon a') + .map((_, elem) => rootUrl + elem.attribs.href) + .get(); + let photo, video; + await Promise.all( + media.map(async (medium) => { + if (medium.includes('/photo/')) { + photo = await fetchPhoto(ctx, medium); + } else if (medium.includes('/video/')) { + video = await fetchVideo(ctx, medium); + } + }) + ); - item.description = art(path.join(__dirname, 'templates/news.art'), { description, photo, video }); + item.description = art(path.join(__dirname, 'templates/news.art'), { description, photo, video }); - return item; - }) - )) { - items.push(item); - } + return item; + }), + { concurrency: 3 } + ); return { title, diff --git a/lib/routes/luogu/contest.ts b/lib/routes/luogu/contest.ts index b89629a2d..a50f39801 100644 --- a/lib/routes/luogu/contest.ts +++ b/lib/routes/luogu/contest.ts @@ -5,7 +5,7 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import MarkdownIt from 'markdown-it'; const md = MarkdownIt(); -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; const baseUrl = 'https://www.luogu.com.cn'; @@ -64,31 +64,31 @@ async function handler() { ) ); - const result = []; - for await (const item of asyncPool(4, data.currentData.contests.result, (item) => - cache.tryGet(`${baseUrl}/contest/${item.id}`, async () => { - const { data: response } = await got(`${baseUrl}/contest/${item.id}`); - const $ = load(response); - const data = JSON.parse( - decodeURIComponent( - $('script') - .text() - .match(/decodeURIComponent\("(.*)"\)/)[1] - ) - ); + const result = await pMap( + data.currentData.contests.result, + (item) => + cache.tryGet(`${baseUrl}/contest/${item.id}`, async () => { + const { data: response } = await got(`${baseUrl}/contest/${item.id}`); + const $ = load(response); + const data = JSON.parse( + decodeURIComponent( + $('script') + .text() + .match(/decodeURIComponent\("(.*)"\)/)[1] + ) + ); - return { - title: item.name, - description: md.render(data.currentData.contest.description), - link: `${baseUrl}/contest/${item.id}`, - author: item.host.name, - pubDate: parseDate(item.startTime, 'X'), - category: [item.rated ? 'Rated' : null, typeMap.ruleType[item.ruleType], typeMap.visibilityType[item.visibilityType]].filter(Boolean), - }; - }) - )) { - result.push(item); - } + return { + title: item.name, + description: md.render(data.currentData.contest.description), + link: `${baseUrl}/contest/${item.id}`, + author: item.host.name, + pubDate: parseDate(item.startTime, 'X'), + category: [item.rated ? 'Rated' : null, typeMap.ruleType[item.ruleType], typeMap.visibilityType[item.visibilityType]].filter(Boolean), + }; + }), + { concurrency: 4 } + ); return { title: $('head title').text(), diff --git a/lib/routes/nextapple/realtime.ts b/lib/routes/nextapple/realtime.ts index 169267ddb..607268152 100644 --- a/lib/routes/nextapple/realtime.ts +++ b/lib/routes/nextapple/realtime.ts @@ -3,7 +3,7 @@ import cache from '@/utils/cache'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; import { load } from 'cheerio'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; export const route: Route = { path: '/realtime/:category?', @@ -41,33 +41,34 @@ async function handler(ctx) { const currentUrl = `https://tw.nextapple.com/realtime/${category}`; const response = await got(currentUrl); const $ = load(response.data); - const items = []; - for await (const item of asyncPool(5, $('article.infScroll'), (item) => { - const link = $(item).find('.post-title').attr('href'); - return cache.tryGet(link, async () => { - const response = await got(link); - const $ = load(response.data); - const mainContent = $('#main-content'); - const titleElement = mainContent.find('header h1'); - const title = titleElement.text(); - titleElement.remove(); - const postMetaElement = mainContent.find('.post-meta'); - const category = postMetaElement.find('.category').text(); - const pubDate = parseDate(postMetaElement.find('time').attr('datetime')); - postMetaElement.remove(); - $('.post-comments').remove(); + const items = await pMap( + $('article.infScroll').toArray(), + (item) => { + const link = $(item).find('.post-title').attr('href'); + return cache.tryGet(link, async () => { + const response = await got(link); + const $ = load(response.data); + const mainContent = $('#main-content'); + const titleElement = mainContent.find('header h1'); + const title = titleElement.text(); + titleElement.remove(); + const postMetaElement = mainContent.find('.post-meta'); + const category = postMetaElement.find('.category').text(); + const pubDate = parseDate(postMetaElement.find('time').attr('datetime')); + postMetaElement.remove(); + $('.post-comments').remove(); - return { - title, - description: mainContent.html(), - category, - pubDate, - link, - }; - }); - })) { - items.push(item); - } + return { + title, + description: mainContent.html(), + category, + pubDate, + link, + }; + }); + }, + { concurrency: 5 } + ); return { title: $('title').text(), diff --git a/lib/routes/shoppingdesign/posts.ts b/lib/routes/shoppingdesign/posts.ts index f88871180..0e8c4beac 100644 --- a/lib/routes/shoppingdesign/posts.ts +++ b/lib/routes/shoppingdesign/posts.ts @@ -3,7 +3,7 @@ import cache from '@/utils/cache'; import got from '@/utils/got'; import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; export const route: Route = { path: '/posts', @@ -34,31 +34,32 @@ async function handler() { const currentUrl = 'https://www.shoppingdesign.com.tw/post?sn_f=1'; const response = await got(currentUrl); const $ = load(response.data); - const items = []; - // maximum parallel requests on the target website are limited to 11. - for await (const data of asyncPool(10, $('article-item'), (item) => { - item = $(item); - const link = item.attr('url'); - return cache.tryGet(link, async () => { - const response = await got(`${link}?sn_f=1`); - const $ = load(response.data); - const article = $('.left article .htmlview'); - article.find('d-image').each(function () { - $(this).replaceWith(``); - }); + const items = await pMap( + $('article-item').toArray(), + (item) => { + item = $(item); + const link = item.attr('url'); + return cache.tryGet(link, async () => { + const response = await got(`${link}?sn_f=1`); + const $ = load(response.data); + const article = $('.left article .htmlview'); + article.find('d-image').each(function () { + $(this).replaceWith(``); + }); - return { - title: $('.left article .top_info h1').text(), - author: $('meta[name="my:author"]').attr('content'), - description: article.html(), - category: $('meta[name="my:category"]').attr('content'), - pubDate: parseDate($('meta[name="my:publish"]').attr('content')), - link, - }; - }); - })) { - items.push(data); - } + return { + title: $('.left article .top_info h1').text(), + author: $('meta[name="my:author"]').attr('content'), + description: article.html(), + category: $('meta[name="my:category"]').attr('content'), + pubDate: parseDate($('meta[name="my:publish"]').attr('content')), + link, + }; + }); + }, + // maximum parallel requests on the target website are limited to 11. + { concurrency: 10 } + ); return { title: $('meta[property="og:title"]').attr('content'), diff --git a/lib/routes/tfc-taiwan/utils.ts b/lib/routes/tfc-taiwan/utils.ts index b923fd5de..58461edcb 100644 --- a/lib/routes/tfc-taiwan/utils.ts +++ b/lib/routes/tfc-taiwan/utils.ts @@ -2,17 +2,9 @@ import got from '@/utils/got'; import { load } from 'cheerio'; import path from 'node:path'; import { art } from '@/utils/render'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; import { parseDate } from '@/utils/parse-date'; -const asyncPoolAll = async (...args) => { - const results = []; - for await (const result of asyncPool(...args)) { - results.push(result); - } - return results; -}; - const baseUrl = 'https://tfc-taiwan.org.tw'; const parseList = (item) => { @@ -27,26 +19,29 @@ const parseList = (item) => { }; const parseItems = (list, tryGet) => - asyncPoolAll(10, list, (item) => - tryGet(item.link, async () => { - const { data: response } = await got(item.link); - const $ = load(response); + pMap( + list, + (item) => + tryGet(item.link, async () => { + const { data: response } = await got(item.link); + const $ = load(response); - $('.field-name-field-addthis, #fb-root, .fb-comments, .likecoin-embed, style[type="text/css"]').remove(); + $('.field-name-field-addthis, #fb-root, .fb-comments, .likecoin-embed, style[type="text/css"]').remove(); - item.description = art(path.join(__dirname, 'templates/article.art'), { - headerImage: item.image, - content: $('#block-system-main .node-content').html(), - }); + item.description = art(path.join(__dirname, 'templates/article.art'), { + headerImage: item.image, + content: $('#block-system-main .node-content').html(), + }); - item.pubDate = $('meta[property="article:published_time"]').attr('content'); - item.updated = $('meta[property="article:modified_time"]').attr('content'); - item.category = $('.node-tags .field-item') - .toArray() - .map((item) => $(item).text()); + item.pubDate = $('meta[property="article:published_time"]').attr('content'); + item.updated = $('meta[property="article:modified_time"]').attr('content'); + item.category = $('.node-tags .field-item') + .toArray() + .map((item) => $(item).text()); - return item; - }) + return item; + }), + { concurrency: 10 } ); export { baseUrl, parseList, parseItems }; diff --git a/lib/routes/tradingview/blog.ts b/lib/routes/tradingview/blog.ts index 484f9a92a..8ffca34d1 100644 --- a/lib/routes/tradingview/blog.ts +++ b/lib/routes/tradingview/blog.ts @@ -4,7 +4,7 @@ import cache from '@/utils/cache'; import got from '@/utils/got'; import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; import { art } from '@/utils/render'; import path from 'node:path'; @@ -26,7 +26,7 @@ async function handler(ctx) { const $ = load(response); - const items = $('article[id]') + const list = $('article[id]') .slice(0, limit) .toArray() .map((item) => { @@ -55,48 +55,48 @@ async function handler(ctx) { }; }); - for await (const item of asyncPool(3, items, (item) => - cache.tryGet(item.link, async () => { - const { data: detailResponse } = await got(item.link); + const items = await pMap( + list, + (item) => + cache.tryGet(item.link, async () => { + const { data: detailResponse } = await got(item.link); - const content = load(detailResponse); + const content = load(detailResponse); - content('div.entry-content') - .find('img') - .each((_, e) => { - content(e).replaceWith( - art(path.join(__dirname, 'templates/description.art'), { - image: { - src: content(e) - .prop('src') - .replace(/-\d+x\d+\./, '.'), - width: content(e).prop('width'), - height: content(e).prop('height'), - }, - }) - ); + content('div.entry-content') + .find('img') + .each((_, e) => { + content(e).replaceWith( + art(path.join(__dirname, 'templates/description.art'), { + image: { + src: content(e) + .prop('src') + .replace(/-\d+x\d+\./, '.'), + width: content(e).prop('width'), + height: content(e).prop('height'), + }, + }) + ); + }); + + item.title = content('meta[property="og:title"]').prop('content'); + item.description = art(path.join(__dirname, 'templates/description.art'), { + image: { + src: content('meta[property="og:image"]').prop('content'), + alt: item.title, + }, + description: content('div.entry-content').html(), }); + item.author = content('meta[property="og:site_name"]').prop('content'); + item.category = content('div.sections a.section') + .toArray() + .map((c) => content(c).text()); + item.pubDate = parseDate(content('div.single-date').text(), 'MMM D, YYYY'); - item.title = content('meta[property="og:title"]').prop('content'); - item.description = art(path.join(__dirname, 'templates/description.art'), { - image: { - src: content('meta[property="og:image"]').prop('content'), - alt: item.title, - }, - description: content('div.entry-content').html(), - }); - item.author = content('meta[property="og:site_name"]').prop('content'); - item.category = content('div.sections a.section') - .toArray() - .map((c) => content(c).text()); - item.pubDate = parseDate(content('div.single-date').text(), 'MMM D, YYYY'); - - return item; - }) - )) { - items.shift(); - items.push(item); - } + return item; + }), + { concurrency: 3 } + ); const icon = new URL($('link[rel="icon"]').prop('href'), rootUrl).href; diff --git a/lib/routes/wsj/news.ts b/lib/routes/wsj/news.ts index 801ccadf4..01b7998c9 100644 --- a/lib/routes/wsj/news.ts +++ b/lib/routes/wsj/news.ts @@ -1,7 +1,8 @@ import { Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { asyncPoolAll, parseArticle } from './utils'; +import { parseArticle } from './utils'; +import pMap from 'p-map'; const hostMap = { 'en-us': 'https://www.wsj.com', 'zh-cn': 'https://cn.wsj.com/zh-hans', @@ -72,7 +73,7 @@ async function handler(ctx) { item.test = key; return item; }); - const items = await asyncPoolAll(10, list, (item) => parseArticle(item)); + const items = await pMap(list, (item) => parseArticle(item), { concurrency: 10 }); return { title: `WSJ${subTitle}`, diff --git a/lib/routes/wsj/utils.ts b/lib/routes/wsj/utils.ts index 4310695ba..1e3507878 100644 --- a/lib/routes/wsj/utils.ts +++ b/lib/routes/wsj/utils.ts @@ -1,5 +1,4 @@ import cache from '@/utils/cache'; -import asyncPool from 'tiny-async-pool'; import { load } from 'cheerio'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; @@ -111,11 +110,4 @@ const parseArticle = (item) => }; }); -const asyncPoolAll = async (...args) => { - const results = []; - for await (const result of asyncPool(...args)) { - results.push(result); - } - return results; -}; -export { asyncPoolAll, parseArticle }; +export { parseArticle }; diff --git a/lib/routes/x-mol/paper.ts b/lib/routes/x-mol/paper.ts index 69c01732f..079283874 100644 --- a/lib/routes/x-mol/paper.ts +++ b/lib/routes/x-mol/paper.ts @@ -5,7 +5,7 @@ import { load } from 'cheerio'; import utils from './utils'; import { parseDate } from '@/utils/parse-date'; import timezone from '@/utils/timezone'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; export const route: Route = { path: '/paper/:type/:magazine', @@ -57,31 +57,26 @@ async function handler(ctx) { }; }); - const asyncPoolAll = async (...args) => { - const results = []; - for await (const result of asyncPool(...args)) { - results.push(result); - } - return results; - }; + const item = await pMap( + newsItem, + (element) => + cache.tryGet(element.link, async () => { + const response = await got(element.link); + const $ = load(response.data); - const item = await asyncPoolAll(2, newsItem, (element) => - cache.tryGet(element.link, async () => { - const response = await got(element.link); - const $ = load(response.data); + const description = $('.maga-content'); + element.doi = description.find('.itsmblue').eq(1).text().trim(); - const description = $('.maga-content'); - element.doi = description.find('.itsmblue').eq(1).text().trim(); + description.find('.itgaryfirst').remove(); + description.find('span').eq(0).remove(); + element.author = description.find('span').eq(0).text().trim(); + description.find('span').eq(0).remove(); - description.find('.itgaryfirst').remove(); - description.find('span').eq(0).remove(); - element.author = description.find('span').eq(0).text().trim(); - description.find('span').eq(0).remove(); + element.description = description.html(); - element.description = description.html(); - - return element; - }) + return element; + }), + { concurrency: 2 } ); return { diff --git a/lib/routes/yamibo/bbs/forum.ts b/lib/routes/yamibo/bbs/forum.ts index 21820def9..802685b77 100644 --- a/lib/routes/yamibo/bbs/forum.ts +++ b/lib/routes/yamibo/bbs/forum.ts @@ -3,7 +3,8 @@ import type { Context } from 'hono'; import { config } from '@/config'; import ofetch from '@/utils/ofetch'; import { load } from 'cheerio'; -import { asyncPoolAll, fetchThread, generateDescription, getDate, bbsOrigin } from '../utils'; +import { fetchThread, generateDescription, getDate, bbsOrigin } from '../utils'; +import pMap from 'p-map'; import cache from '@/utils/cache'; export const route: Route = { @@ -81,8 +82,7 @@ async function handler(ctx: Context): Promise { }; }); - items = await asyncPoolAll( - 5, + items = await pMap( items, async (item) => (await cache.tryGet(item.link!, async () => { @@ -105,7 +105,8 @@ async function handler(ctx: Context): Promise { description, pubDate: item.pubDate, }; - })) as DataItem + })) as DataItem, + { concurrency: 5 } ); return { diff --git a/lib/routes/yamibo/utils.ts b/lib/routes/yamibo/utils.ts index 3e6a7ee0e..147887786 100644 --- a/lib/routes/yamibo/utils.ts +++ b/lib/routes/yamibo/utils.ts @@ -2,7 +2,6 @@ import { parseDate } from '@/utils/parse-date'; import timezone from '@/utils/timezone'; import ofetch from '@/utils/ofetch'; import { config } from '@/config'; -import asyncPool from 'tiny-async-pool'; import { JSDOM } from 'jsdom'; import type { Cheerio, Element } from 'cheerio'; @@ -104,11 +103,3 @@ export function generateDescription($item: Cheerio, postId: string) { return description; } - -export async function asyncPoolAll(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise) { - const results: Awaited = []; - for await (const result of asyncPool(poolLimit, array, iteratorFn)) { - results.push(result); - } - return results; -} diff --git a/lib/routes/youtube/subscriptions.ts b/lib/routes/youtube/subscriptions.ts index 7455f30be..c6497268e 100644 --- a/lib/routes/youtube/subscriptions.ts +++ b/lib/routes/youtube/subscriptions.ts @@ -3,7 +3,7 @@ import cache from '@/utils/cache'; import { config } from '@/config'; import utils from './utils'; import { parseDate } from '@/utils/parse-date'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { @@ -51,18 +51,10 @@ async function handler(ctx) { const channelIds = (await utils.getSubscriptions('snippet', cache)).data.items.map((item) => item.snippet.resourceId.channelId); - const playlistIds = []; - for await (const playlistId of asyncPool(30, channelIds, async (channelId) => (await utils.getChannelWithId(channelId, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads)) { - playlistIds.push(playlistId); - } + const playlistIds = await pMap(channelIds, async (channelId) => (await utils.getChannelWithId(channelId, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads, { concurrency: 30 }); - let items = []; - for await (const item of asyncPool(30, playlistIds, async (playlistId) => (await utils.getPlaylistItems(playlistId, 'snippet', cache))?.data.items)) { - items.push(item); - } + let items = await pMap(playlistIds, async (playlistId) => (await utils.getPlaylistItems(playlistId, 'snippet', cache))?.data.items, { concurrency: 30 }); - // https://measurethat.net/Benchmarks/Show/7223 - // concat > reduce + concat >>> flat items = items.flat(); items = items diff --git a/lib/routes/zaker/channel.ts b/lib/routes/zaker/channel.ts index e4e6f536f..c3e68682a 100644 --- a/lib/routes/zaker/channel.ts +++ b/lib/routes/zaker/channel.ts @@ -2,7 +2,7 @@ import { Route } from '@/types'; import cache from '@/utils/cache'; import * as cheerio from 'cheerio'; import { baseUrl, fetchItem, getSafeLineCookieWithData, parseList } from './utils'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; export const route: Route = { path: '/channel/:id?', @@ -31,10 +31,7 @@ async function handler(ctx) { const feedTitle = $('head title').text(); const list = parseList($); - const items = []; - for await (const item of asyncPool(2, list, (item) => cache.tryGet(item.link!, () => fetchItem(item, cookie)))) { - items.push(item); - } + const items = await pMap(list, (item) => cache.tryGet(item.link!, () => fetchItem(item, cookie)), { concurrency: 2 }); return { title: feedTitle, diff --git a/lib/routes/zaker/focus.ts b/lib/routes/zaker/focus.ts index 4349f9d2a..539b838f2 100644 --- a/lib/routes/zaker/focus.ts +++ b/lib/routes/zaker/focus.ts @@ -1,7 +1,7 @@ import { Route } from '@/types'; import cache from '@/utils/cache'; import * as cheerio from 'cheerio'; -import asyncPool from 'tiny-async-pool'; +import pMap from 'p-map'; import { baseUrl, fetchItem, getSafeLineCookieWithData, parseList } from './utils'; export const route: Route = { @@ -26,10 +26,7 @@ async function handler() { const $ = cheerio.load(data); const list = parseList($); - const items = []; - for await (const item of asyncPool(2, list, (item) => cache.tryGet(item.link!, () => fetchItem(item, cookie)))) { - items.push(item); - } + const items = await pMap(list, (item) => cache.tryGet(item.link!, () => fetchItem(item, cookie)), { concurrency: 2 }); return { title: 'ZAKER 精读新闻', diff --git a/package.json b/package.json index af2c24ebb..6b3aee5e4 100644 --- a/package.json +++ b/package.json @@ -125,7 +125,6 @@ "socks-proxy-agent": "8.0.5", "source-map": "0.7.4", "telegram": "2.26.22", - "tiny-async-pool": "2.1.0", "title": "4.0.1", "tldts": "7.0.2", "tosource": "2.0.0-alpha.3", @@ -166,7 +165,6 @@ "@types/node": "22.14.1", "@types/sanitize-html": "2.15.0", "@types/supertest": "6.0.3", - "@types/tiny-async-pool": "2.0.3", "@types/title": "3.4.3", "@types/uuid": "10.0.0", "@typescript-eslint/eslint-plugin": "8.31.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 459eee3d7..929d683d3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -242,9 +242,6 @@ importers: telegram: specifier: 2.26.22 version: 2.26.22 - tiny-async-pool: - specifier: 2.1.0 - version: 2.1.0 title: specifier: 4.0.1 version: 4.0.1 @@ -360,9 +357,6 @@ importers: '@types/supertest': specifier: 6.0.3 version: 6.0.3 - '@types/tiny-async-pool': - specifier: 2.0.3 - version: 2.0.3 '@types/title': specifier: 3.4.3 version: 3.4.3 @@ -2534,9 +2528,6 @@ packages: '@types/tedious@4.0.14': resolution: {integrity: sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==} - '@types/tiny-async-pool@2.0.3': - resolution: {integrity: sha512-n3l1s538tKo9RBoHs4I3DG/VmD3VYhF5mHcgu1sU4Lq7JCNBtxnpBy3OkWSbZsp5r5QOuplh2UkXXXwufoAuNQ==} - '@types/title@3.4.3': resolution: {integrity: sha512-mjupLOb4kwUuoUFokkacy/VMRVBH2qtqZ5AX7K7iha6+iKIkX80n/Y4EoNVEVRmer8dYJU/ry+fppUaDFVQh7Q==} @@ -5881,9 +5872,6 @@ packages: through@2.3.8: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} - tiny-async-pool@2.1.0: - resolution: {integrity: sha512-ltAHPh/9k0STRQqaoUX52NH4ZQYAJz24ZAEwf1Zm+HYg3l9OXTWeqWKyYsHu40wF/F0rxd2N2bk5sLvX2qlSvg==} - tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -8646,8 +8634,6 @@ snapshots: dependencies: '@types/node': 22.14.1 - '@types/tiny-async-pool@2.0.3': {} - '@types/title@3.4.3': {} '@types/tough-cookie@4.0.5': {} @@ -12483,8 +12469,6 @@ snapshots: through@2.3.8: {} - tiny-async-pool@2.1.0: {} - tinybench@2.9.0: {} tinyexec@0.3.2: {}