From cfcdc34e1f5fe11b91840975ddf188824e796b7e Mon Sep 17 00:00:00 2001 From: Tony Date: Sun, 7 Jun 2026 02:43:20 +0800 Subject: [PATCH] refactor: replace playwright with patchright (#21976) * refactor: use patchwright * refactor: migrate setRequestInterception and waitUntil * fix: overwrite ua * refactor: drop puppeteer cookie/requestfinished shims Migrate page.cookies/setCookie and browser.cookies/setCookie call sites to the native context APIs (page.context().cookies/addCookies), rewrite requestfinished handlers to await request.response() instead of consuming the synchronous puppeteer-shaped payload, and remove the now-redundant browser.close context-cleanup wrapper. Playwright already handles all of these natively; the shims existed only for the puppeteer -> playwright migration. * refactor: remove page.authenticate, page.setUserAgent, browser.newPage(), browser.userAgent() shims update cookie utility tests * refactor: replace browser instance with context in Playwright usage across multiple routes - Updated all instances of `browser` to `context` in Playwright-related functions for consistency and improved resource management. - Ensured proper closure of context after operations to prevent memory leaks. - Adjusted request interception methods to utilize the new context variable. * fix: direct type import * refactor: remove Puppeteer compatibility layers and unify page handling with Playwright * fix: test case * chore: bump patchright to v1.60.0 * chore: pin patchright to 1.59.1 nft can't resolve browser.json due to https://github.com/microsoft/playwright/pull/40093 * chore: pin to node 24.15 for playwright browser install on node 24.16 stucks due to https://github.com/microsoft/playwright/issues/40724 * fix: update patchright and patchright-core to version 1.60.2 and adjust minify-docker.js for compatibility * fix: import BrowserContext from patchright --- .github/workflows/issue-command.yml | 2 +- .github/workflows/test.yml | 4 +- Dockerfile | 10 +- lib/bilibili-video-route.test.ts | 11 +- lib/routes/acs/journal.tsx | 12 +- lib/routes/aip/journal-pupp.ts | 6 +- lib/routes/aip/utils.tsx | 10 +- lib/routes/alternativeto/utils.ts | 12 +- lib/routes/apkpure/versions.ts | 12 +- lib/routes/bilibili/cache.ts | 2 +- lib/routes/bilibili/video.ts | 8 +- lib/routes/bilibili/wasm-exec.ts | 4 +- lib/routes/bluestacks/release.ts | 20 +- lib/routes/ccac/news.ts | 12 +- lib/routes/chinadegrees/province.tsx | 12 +- lib/routes/chinatimes/index.ts | 12 +- lib/routes/cjlu/yjsy/index.ts | 13 +- lib/routes/cmde/index.ts | 20 +- lib/routes/colamanga/manga.ts | 13 +- lib/routes/cw/author.ts | 6 +- lib/routes/cw/master.ts | 6 +- lib/routes/cw/sub.ts | 6 +- lib/routes/cw/today.ts | 6 +- lib/routes/cw/utils.ts | 31 +-- lib/routes/dailypush/all.ts | 8 +- lib/routes/dailypush/tags.ts | 8 +- lib/routes/dailypush/utils.ts | 21 +- lib/routes/dcard/section.ts | 16 +- lib/routes/dcard/utils.ts | 14 +- lib/routes/douyin/hashtag.ts | 14 +- lib/routes/douyin/live.ts | 15 +- lib/routes/douyin/user.ts | 15 +- lib/routes/fortnite/news.ts | 15 +- lib/routes/gov/customs/list.ts | 8 +- lib/routes/gov/customs/utils.ts | 10 +- lib/routes/gov/hangzhou/zjzwfw.ts | 14 +- lib/routes/gov/hangzhou/zwfw.tsx | 6 +- lib/routes/gov/pbc/goutongjiaoliu.ts | 20 +- lib/routes/gov/pbc/trade-announcement.ts | 20 +- lib/routes/hitcon/zeroday.tsx | 13 +- lib/routes/hkushop/vinyl-or-picture-lp.ts | 12 +- lib/routes/hottoys/index.ts | 13 +- lib/routes/ielts/index.ts | 12 +- lib/routes/iqiyi/video.ts | 12 +- lib/routes/iwara/ranking.ts | 8 +- lib/routes/javdb/utils.ts | 30 ++- lib/routes/javtrailers/casts.ts | 8 +- lib/routes/javtrailers/categories.ts | 8 +- lib/routes/javtrailers/studios.ts | 8 +- lib/routes/javtrailers/utils.ts | 14 +- lib/routes/kuaishou/profile.ts | 15 +- lib/routes/linkedin/posts.ts | 13 +- lib/routes/missav/new.tsx | 12 +- lib/routes/nhentai/util.tsx | 6 +- lib/routes/njust/utils.ts | 14 +- lib/routes/nuaa/utils/pypasswaf.ts | 14 +- lib/routes/nytimes/index.ts | 6 +- lib/routes/nytimes/utils.ts | 10 +- lib/routes/oceanengine/arithmetic-index.tsx | 12 +- lib/routes/parliament.uk/commonslibrary.ts | 12 +- lib/routes/parliament.uk/lordslibrary.ts | 12 +- lib/routes/perplexity/blog.ts | 16 +- lib/routes/perplexity/changelog.ts | 16 +- lib/routes/picnob/utils.ts | 6 +- lib/routes/picuki/profile.ts | 6 +- lib/routes/pincong/utils.ts | 12 +- lib/routes/pnas/index.tsx | 12 +- lib/routes/researchgate/publications.ts | 20 +- lib/routes/science/blogs.ts | 13 +- lib/routes/science/current.ts | 6 +- lib/routes/science/early.ts | 6 +- lib/routes/science/utils.tsx | 10 +- lib/routes/sotwe/user.ts | 12 +- lib/routes/spankbang/new-videos.tsx | 12 +- lib/routes/tiktok/user.ts | 14 +- lib/routes/toutiao/a-bogus.ts | 4 +- lib/routes/twitter/api/web-api/login.ts | 8 +- lib/routes/uchicago/current.ts | 20 +- lib/routes/uestc/auto.ts | 14 +- lib/routes/uestc/cqe.ts | 14 +- lib/routes/uestc/scse.ts | 14 +- lib/routes/uestc/sice.ts | 14 +- lib/routes/uestc/sise.ts | 14 +- lib/routes/ups/track.ts | 16 +- lib/routes/uraaka-joshi/uraaka-joshi-user.ts | 18 +- lib/routes/uraaka-joshi/uraaka-joshi.ts | 18 +- lib/routes/weibo/utils.ts | 13 +- lib/routes/xiaohongshu/util.ts | 18 +- lib/routes/xsijishe/rank.ts | 6 +- lib/routes/xsijishe/utils.ts | 15 +- lib/routes/xueqiu/cookies.ts | 10 +- lib/routes/xueqiu/user.ts | 6 +- lib/routes/zhihu/execlib/x-zse-96-v3.ts | 4 +- lib/utils/playwright-utils.test.ts | 40 +-- lib/utils/playwright-utils.ts | 6 +- lib/utils/playwright.mock.test.ts | 133 +++------- lib/utils/playwright.test.ts | 31 +-- lib/utils/playwright.ts | 262 ++++--------------- lib/utils/playwright.worker.ts | 199 ++------------ lib/utils/puppeteer.ts | 2 - lib/utils/puppeteer.worker.ts | 2 - package.json | 5 +- pnpm-lock.yaml | 42 +-- scripts/docker/minify-docker.js | 14 + 104 files changed, 730 insertions(+), 1111 deletions(-) delete mode 100644 lib/utils/puppeteer.ts delete mode 100644 lib/utils/puppeteer.worker.ts diff --git a/.github/workflows/issue-command.yml b/.github/workflows/issue-command.yml index e56334150..9f4c99ac5 100644 --- a/.github/workflows/issue-command.yml +++ b/.github/workflows/issue-command.yml @@ -120,7 +120,7 @@ jobs: cache: 'pnpm' - name: Install dependencies (pnpm) - run: pnpm i && pnpm rb && pnpm exec playwright install chromium + run: pnpm i && pnpm rb && pnpm exec patchright install chromium - name: Fetch affected routes id: fetch-route diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05b04ca7a..d641221ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: - name: Install dependencies (pnpm) run: pnpm i - name: Run postinstall script for dependencies - run: pnpm rb && pnpm exec playwright install chromium + run: pnpm rb && pnpm exec patchright install chromium - name: Build routes run: pnpm build - name: Build worker routes @@ -90,7 +90,7 @@ jobs: run: pnpm build - name: Install bundled Chromium if: ${{ matrix.chromium.dependency == '' }} - run: pnpm exec playwright install chromium + run: pnpm exec patchright install chromium - name: Install Chromium if: ${{ matrix.chromium.dependency != '' }} # 'chromium-browser' from Ubuntu APT repo is a dummy package. Its version (85.0.4183.83) means diff --git a/Dockerfile b/Dockerfile index 6683193b7..0654acb99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ WORKDIR /ver COPY ./package.json /app/ RUN \ set -ex && \ - grep -Po '(?<="playwright": ")[^\s"]*(?=")' /app/package.json | tee /ver/.playwright_version && \ + grep -Po '(?<="patchright": ")[^\s"]*(?=")' /app/package.json | tee /ver/.patchright_version && \ grep -Po '(?<="@vercel/nft": ")[^\s"]*(?=")' /app/package.json | tee /ver/.nft_version && \ grep -Po '(?<="fs-extra": ")[^\s"]*(?=")' /app/package.json | tee /ver/.fs_extra_version @@ -88,12 +88,12 @@ FROM node:24-bookworm-slim AS chromium-downloader # Yeah, downloading Chromium never needs those dependencies below. WORKDIR /app -COPY --from=dep-version-parser /ver/.playwright_version /app/.playwright_version +COPY --from=dep-version-parser /ver/.patchright_version /app/.patchright_version ARG TARGETPLATFORM ARG USE_CHINA_NPM_REGISTRY=0 ARG PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 -# The official recommended way to use Playwright on x86(_64) is to use the bundled browser. +# The official recommended way to use Patchright on x86(_64) is to use the bundled browser. RUN \ set -ex ; \ if [ "$PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD" = 0 ] && [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \ @@ -106,9 +106,9 @@ RUN \ unset PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD && \ export PLAYWRIGHT_BROWSERS_PATH=/app/node_modules/.cache/ms-playwright && \ corepack enable pnpm && \ - pnpm --allow-build=playwright add playwright@$(cat /app/.playwright_version) --save-prod && \ + pnpm --allow-build=patchright --allow-build=patchright-core add patchright@$(cat /app/.patchright_version) --save-prod && \ pnpm rb && \ - pnpm exec playwright install chromium ; \ + pnpm exec patchright install chromium ; \ else \ mkdir -p /app/node_modules/.cache/ms-playwright ; \ fi; diff --git a/lib/bilibili-video-route.test.ts b/lib/bilibili-video-route.test.ts index 09f601237..39dd7ea1b 100644 --- a/lib/bilibili-video-route.test.ts +++ b/lib/bilibili-video-route.test.ts @@ -11,15 +11,13 @@ const destroy = vi.fn(); const getPlaywrightPage = vi.fn(); const goto = vi.fn(); const on = vi.fn(); -const setCookie = vi.fn(); -const setRequestInterception = vi.fn(); +const pageRoute = vi.fn(); const waitForResponse = vi.fn(); const page = { goto, on, - setCookie, - setRequestInterception, + route: pageRoute, waitForResponse, }; @@ -66,8 +64,7 @@ describe('/bilibili/user/video/:uid', () => { getPlaywrightPage.mockReset(); goto.mockReset(); on.mockReset(); - setCookie.mockReset(); - setRequestInterception.mockReset(); + pageRoute.mockReset(); waitForResponse.mockReset(); }); @@ -109,6 +106,7 @@ describe('/bilibili/user/video/:uid', () => { getPlaywrightPage.mockImplementation(async (_url, options) => { await options.onBeforeLoad?.(page); return { + context: {}, destroy, page, }; @@ -171,6 +169,7 @@ describe('/bilibili/user/video/:uid', () => { getPlaywrightPage.mockImplementation(async (_url, options) => { await options.onBeforeLoad?.(page); return { + context: {}, destroy, page, }; diff --git a/lib/routes/acs/journal.tsx b/lib/routes/acs/journal.tsx index 4ad4d5587..d5f2b8478 100644 --- a/lib/routes/acs/journal.tsx +++ b/lib/routes/acs/journal.tsx @@ -28,14 +28,14 @@ async function handler(ctx) { let title = ''; - const browser = await playwright(); + const context = await playwright(); const items = await cache.tryGet( currentUrl, async () => { - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); await page.goto(currentUrl, { waitUntil: 'domcontentloaded', @@ -76,7 +76,7 @@ async function handler(ctx) { false ); - await browser.close(); + await context.close(); return { title, diff --git a/lib/routes/aip/journal-pupp.ts b/lib/routes/aip/journal-pupp.ts index 845661a89..3e0412b5a 100644 --- a/lib/routes/aip/journal-pupp.ts +++ b/lib/routes/aip/journal-pupp.ts @@ -18,12 +18,12 @@ const handler = async (ctx) => { } // use Playwright due to the obstacle by cloudflare challenge - const browser = await playwright(); + const context = await playwright(); const { jrnlName, list } = await cache.tryGet( jrnlUrl, async () => { - const response = await playwrightGet(jrnlUrl, browser); + const response = await playwrightGet(jrnlUrl, context); const $ = load(response); const jrnlName = $('.header-journal-title').text(); const list = $('.card') @@ -52,7 +52,7 @@ const handler = async (ctx) => { false ); - await browser.close(); + await context.close(); return { title: jrnlName, diff --git a/lib/routes/aip/utils.tsx b/lib/routes/aip/utils.tsx index ba592557f..00826b987 100644 --- a/lib/routes/aip/utils.tsx +++ b/lib/routes/aip/utils.tsx @@ -1,11 +1,11 @@ import { renderToString } from 'hono/jsx/dom/server'; -const playwrightGet = async (url, browser) => { - const page = await browser.newPage(); +const playwrightGet = async (url, context) => { + const page = await context.newPage(); // await page.setExtraHTTPHeaders({ referer: host }); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' ? request.continue() : request.abort(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' ? route.continue() : route.abort(); }); await page.goto(url, { waitUntil: 'domcontentloaded', diff --git a/lib/routes/alternativeto/utils.ts b/lib/routes/alternativeto/utils.ts index 08fd66814..8160f19c5 100644 --- a/lib/routes/alternativeto/utils.ts +++ b/lib/routes/alternativeto/utils.ts @@ -4,17 +4,17 @@ const baseURL = 'https://alternativeto.net'; const playwrightGet = (url, cache) => cache.tryGet(url, async () => { - const browser = await playwright(); - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' ? request.continue() : request.abort(); + const context = await playwright(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' ? route.continue() : route.abort(); }); await page.goto(url, { waitUntil: 'domcontentloaded', }); const html = await page.evaluate(() => document.documentElement.innerHTML); - await browser.close(); + await context.close(); return html; }); diff --git a/lib/routes/apkpure/versions.ts b/lib/routes/apkpure/versions.ts index 911f05d2a..bce6bb781 100644 --- a/lib/routes/apkpure/versions.ts +++ b/lib/routes/apkpure/versions.ts @@ -28,11 +28,11 @@ async function handler(ctx) { const baseUrl = 'https://apkpure.com'; const link = `${baseUrl}/${region}/${pkg}/versions`; - const browser = await playwright(); - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' ? request.continue() : request.abort(); + const context = await playwright(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' ? route.continue() : route.abort(); }); logger.http(`Requesting ${link}`); await page.goto(link, { @@ -40,7 +40,7 @@ async function handler(ctx) { }); const r = await page.evaluate(() => document.documentElement.innerHTML); - await browser.close(); + await context.close(); const $ = load(r); const img = new URL($('.ver-top img').attr('src')); diff --git a/lib/routes/bilibili/cache.ts b/lib/routes/bilibili/cache.ts index c37546af9..549f8680c 100644 --- a/lib/routes/bilibili/cache.ts +++ b/lib/routes/bilibili/cache.ts @@ -51,7 +51,7 @@ const getCookie = (disableConfig = false) => { waitForRequest = new Promise((resolve) => { page.on('requestfinished', async (request) => { if (request.url() === 'https://api.bilibili.com/x/web-interface/nav') { - const cookies = await page.cookies(); + const cookies = await page.context().cookies(); let cookieString = cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; '); cookieString = cookieString.replace(/b_lsid=[0-9A-F]+_[0-9A-F]+/, `b_lsid=${utils.lsid()}`); resolve(cookieString); diff --git a/lib/routes/bilibili/video.ts b/lib/routes/bilibili/video.ts index 9b2a66bdf..c34a821c3 100644 --- a/lib/routes/bilibili/video.ts +++ b/lib/routes/bilibili/video.ts @@ -140,7 +140,7 @@ async function applyCookie(page: Page, cookie: string) { .filter((item) => item !== undefined); if (cookies.length > 0) { - await page.setCookie(...cookies); + await page.context().addCookies(cookies); } } @@ -184,9 +184,9 @@ async function fetchVideoListFromBrowser(uid: string): Promise { await applyCookie(page, cookie); } - await page.setRequestInterception(true); - page.on('request', (request) => { - allowedBrowserRequestTypes.has(request.resourceType()) ? request.continue() : request.abort(); + await page.route('**/*', (route) => { + const request = route.request(); + allowedBrowserRequestTypes.has(request.resourceType()) ? route.continue() : route.abort(); }); }, gotoConfig: { waitUntil: 'domcontentloaded' }, diff --git a/lib/routes/bilibili/wasm-exec.ts b/lib/routes/bilibili/wasm-exec.ts index b4359a3da..2ed2fd0e0 100644 --- a/lib/routes/bilibili/wasm-exec.ts +++ b/lib/routes/bilibili/wasm-exec.ts @@ -2,7 +2,7 @@ // oxlint-disable no-unused-vars /* eslint-disable prefer-rest-params */ /* eslint-disable default-case */ -/* eslint-disable unicorn/consistent-function-scoping */ +// oxlint-disable unicorn/consistent-function-scoping /* eslint-disable no-console */ // Copyright 2018 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -634,7 +634,7 @@ _makeFuncWrapper(id) { // somehow avoiding aliasing this with an arrow function doesn't work - // eslint-disable-next-line unicorn/no-this-assignment, @typescript-eslint/no-this-alias + // oxlint-disable-next-line unicorn/no-this-assignment typescript/no-this-alias const go = this; return function () { const event = { id, this: this, args: arguments }; diff --git a/lib/routes/bluestacks/release.ts b/lib/routes/bluestacks/release.ts index bf3c823ec..19fc5b164 100644 --- a/lib/routes/bluestacks/release.ts +++ b/lib/routes/bluestacks/release.ts @@ -32,11 +32,11 @@ export const route: Route = { }; async function handler() { - const browser = await playwright(); - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + const context = await playwright(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); await page.goto(pageUrl, { waitUntil: 'domcontentloaded', @@ -59,10 +59,10 @@ async function handler() { await Promise.all( items.map((item) => cache.tryGet(item.link, async () => { - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); await page.goto(item.link, { waitUntil: 'domcontentloaded', @@ -79,7 +79,7 @@ async function handler() { ) ); - await browser.close(); + await context.close(); return { title: $('.article__title').text().trim(), diff --git a/lib/routes/ccac/news.ts b/lib/routes/ccac/news.ts index cf893ee8e..508c196ce 100644 --- a/lib/routes/ccac/news.ts +++ b/lib/routes/ccac/news.ts @@ -32,21 +32,21 @@ export const route: Route = { }; async function handler(ctx) { - const browser = await playwright(); + const context = await playwright(); const lang = ctx.req.param('lang') ?? 'sc'; const type = utils.TYPE[ctx.req.param('type')]; const BASE = utils.langBase(lang); - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); await page.goto(BASE, { waitUntil: 'domcontentloaded', }); const articles = await page.evaluate(() => window.articles); - await browser.close(); + await context.close(); const list = utils .typeFilter(articles, type) diff --git a/lib/routes/chinadegrees/province.tsx b/lib/routes/chinadegrees/province.tsx index e32c38d7d..e129324fe 100644 --- a/lib/routes/chinadegrees/province.tsx +++ b/lib/routes/chinadegrees/province.tsx @@ -82,11 +82,11 @@ async function handler(ctx) { const data = await cache.tryGet( url, async () => { - const browser = await playwright(); - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + const context = await playwright(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); await page.goto(url, { waitUntil: 'domcontentloaded', @@ -94,7 +94,7 @@ async function handler(ctx) { await page.waitForSelector('.datalist'); const html = await page.evaluate(() => document.documentElement.innerHTML); - await browser.close(); + await context.close(); const $ = load(html); return { diff --git a/lib/routes/chinatimes/index.ts b/lib/routes/chinatimes/index.ts index bfeb05a87..20c9bdab7 100644 --- a/lib/routes/chinatimes/index.ts +++ b/lib/routes/chinatimes/index.ts @@ -43,7 +43,7 @@ async function handler(ctx) { const response = await ofetch(link); const $ = load(response); - const browser = await playwright(); + const context = await playwright(); const list = $('.articlebox-compact') .toArray() @@ -66,10 +66,10 @@ async function handler(ctx) { const items = await Promise.all( list.map((item) => cache.tryGet(item.link, async () => { - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' ? request.continue() : request.abort(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' ? route.continue() : route.abort(); }); logger.http(`Requesting ${item.link}`); await page.goto(item.link, { @@ -98,7 +98,7 @@ async function handler(ctx) { ) ); - await browser.close(); + await context.close(); return { title: $('head title').text(), diff --git a/lib/routes/cjlu/yjsy/index.ts b/lib/routes/cjlu/yjsy/index.ts index 68804d6e1..8bf31f06a 100644 --- a/lib/routes/cjlu/yjsy/index.ts +++ b/lib/routes/cjlu/yjsy/index.ts @@ -86,19 +86,18 @@ async function handler(ctx) { const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 10; const url = `${host}index/${cate}.htm`; - const { page, destroy, browser } = await getPlaywrightPage(url, { + const { page, destroy } = await getPlaywrightPage(url, { onBeforeLoad: async (page) => { await page.setExtraHTTPHeaders(headers); - await page.setUserAgent(headers['User-Agent']); - await page.setRequestInterception(true); - page.on('request', (request) => { - allowedResourceTypes.has(request.resourceType()) ? request.continue() : request.abort(); + await page.route('**/*', (route) => { + const request = route.request(); + allowedResourceTypes.has(request.resourceType()) ? route.continue() : route.abort(); }); }, - gotoConfig: { waitUntil: 'networkidle2' }, + gotoConfig: { waitUntil: 'networkidle' }, }); - const cookies = await browser.cookies(); + const cookies = await page.context().cookies(); const cookieString = cookies.map((c) => `${c.name}=${c.value}`).join('; '); const response = await page.content(); diff --git a/lib/routes/cmde/index.ts b/lib/routes/cmde/index.ts index 0f7776e45..6e3d8c1cf 100644 --- a/lib/routes/cmde/index.ts +++ b/lib/routes/cmde/index.ts @@ -18,12 +18,12 @@ export const route: Route = { async function handler(ctx) { const cate = ctx.req.param('cate') ?? 'xwdt/zxyw'; const url = `${rootURL}/${cate}/`; - const browser = await playwright(); + const context = await playwright(); const data = await cache.tryGet(url, async () => { - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); await page.goto(url, { waitUntil: 'domcontentloaded', @@ -52,10 +52,10 @@ async function handler(ctx) { const items = await Promise.all( data.items.map((item) => cache.tryGet(item.link, async () => { - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); await page.goto(item.link, { waitUntil: 'domcontentloaded', @@ -72,7 +72,7 @@ async function handler(ctx) { ) ); - await browser.close(); + await context.close(); return { title: data.title, diff --git a/lib/routes/colamanga/manga.ts b/lib/routes/colamanga/manga.ts index 626f1b0d8..bda4c200b 100644 --- a/lib/routes/colamanga/manga.ts +++ b/lib/routes/colamanga/manga.ts @@ -43,14 +43,13 @@ async function handler(ctx: Context) { const id = ctx.req.param('id'); const url = `https://${domain}/${id}`; - const browser = await playwright(); + const context = await playwright(); - const page = await browser.newPage(); + const page = await context.newPage(); - await page.setRequestInterception(true); - - page.on('request', (request) => { - request.resourceType() === 'document' ? request.continue() : request.abort(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' ? route.continue() : route.abort(); }); logger.http(`Requesting ${url}`); @@ -60,7 +59,7 @@ async function handler(ctx: Context) { }); const response = await page.content(); - await browser.close(); + await context.close(); const $ = load(response); diff --git a/lib/routes/cw/author.ts b/lib/routes/cw/author.ts index 2be67a126..fda1501a0 100644 --- a/lib/routes/cw/author.ts +++ b/lib/routes/cw/author.ts @@ -27,11 +27,11 @@ export const route: Route = { }; async function handler(ctx) { - const browser = await playwright(); + const context = await playwright(); - const { $, items } = await parsePage('author', browser, ctx); + const { $, items } = await parsePage('author', context, ctx); - await browser.close(); + await context.close(); return { title: $('head title').text(), diff --git a/lib/routes/cw/master.ts b/lib/routes/cw/master.ts index eaae855f3..239d6c22a 100644 --- a/lib/routes/cw/master.ts +++ b/lib/routes/cw/master.ts @@ -37,11 +37,11 @@ export const route: Route = { }; async function handler(ctx) { - const browser = await playwright(); + const context = await playwright(); - const { $, items } = await parsePage('master', browser, ctx); + const { $, items } = await parsePage('master', context, ctx); - await browser.close(); + await context.close(); return { title: $('head title').text(), diff --git a/lib/routes/cw/sub.ts b/lib/routes/cw/sub.ts index 3f5abdb72..a374c6d19 100644 --- a/lib/routes/cw/sub.ts +++ b/lib/routes/cw/sub.ts @@ -22,11 +22,11 @@ export const route: Route = { }; async function handler(ctx) { - const browser = await playwright(); + const context = await playwright(); - const { $, items } = await parsePage('sub', browser, ctx); + const { $, items } = await parsePage('sub', context, ctx); - await browser.close(); + await context.close(); return { title: $('head title').text(), diff --git a/lib/routes/cw/today.ts b/lib/routes/cw/today.ts index dd3405633..73e4303f4 100644 --- a/lib/routes/cw/today.ts +++ b/lib/routes/cw/today.ts @@ -28,11 +28,11 @@ export const route: Route = { }; async function handler(ctx) { - const browser = await playwright(); + const context = await playwright(); - const { $, items } = await parsePage('today', browser, ctx); + const { $, items } = await parsePage('today', context, ctx); - await browser.close(); + await context.close(); return { title: $('head title').text(), diff --git a/lib/routes/cw/utils.ts b/lib/routes/cw/utils.ts index f1f9a9859..be75a2aaf 100644 --- a/lib/routes/cw/utils.ts +++ b/lib/routes/cw/utils.ts @@ -1,5 +1,6 @@ import { load } from 'cheerio'; +import { config } from '@/config'; import cache from '@/utils/cache'; import logger from '@/utils/logger'; import ofetch from '@/utils/ofetch'; @@ -29,13 +30,13 @@ const pathMap = { }, }; -const getCookie = async (browser, tryGet) => { +const getCookie = async (context, tryGet) => { if (!cookie) { cookie = await tryGet('cw:cookie', async () => { - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); logger.http(`Requesting ${baseUrl}/user/get/cookie-bar`); await page.goto(`${baseUrl}/user/get/cookie-bar`, { @@ -49,14 +50,14 @@ const getCookie = async (browser, tryGet) => { return cookie; }; -const parsePage = async (path, browser, ctx) => { +const parsePage = async (path, context, ctx) => { const pageUrl = `${baseUrl}${pathMap[path].pageUrl(ctx.req.param('channel'))}`; - const cookie = await getCookie(browser, cache.tryGet); - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + const cookie = await getCookie(context, cache.tryGet); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); await setCookies(page, cookie, 'cw.com.tw'); logger.http(`Requesting ${pageUrl}`); @@ -70,7 +71,7 @@ const parsePage = async (path, browser, ctx) => { const $ = load(response); const list = parseList($, ctx.req.query('limit') ? Number(ctx.req.query('limit')) : pathMap[path].limit); - const items = await parseItems(list, browser, cache.tryGet); + const items = await parseItems(list, context, cache.tryGet); return { $, items }; }; @@ -88,14 +89,14 @@ const parseList = ($, limit) => }) .slice(0, limit); -const parseItems = (list, browser, tryGet) => +const parseItems = (list, context, tryGet) => Promise.all( list.map((item) => tryGet(item.link, async () => { const response = await ofetch(item.link, { headers: { - Cookie: await getCookie(browser, tryGet), - 'User-Agent': browser.userAgent(), + Cookie: await getCookie(context, tryGet), + 'User-Agent': config.ua, }, }); const $ = load(response); diff --git a/lib/routes/dailypush/all.ts b/lib/routes/dailypush/all.ts index 6a39fe133..e92405045 100644 --- a/lib/routes/dailypush/all.ts +++ b/lib/routes/dailypush/all.ts @@ -42,12 +42,12 @@ async function handler(ctx) { const { sort = '' } = ctx.req.param(); const url = sort ? `${BASE_URL}/${sort}` : BASE_URL; - const browser = await playwright(); + const context = await playwright(); try { - const html = await fetchPageHtml(browser, url, 'article'); + const html = await fetchPageHtml(context, url, 'article'); const $ = load(html); const list = parseArticles($, BASE_URL); - const items = await enhanceItemsWithSummaries(browser, list); + const items = await enhanceItemsWithSummaries(context, list); const pageTitle = $('title').text() || 'DailyPush - All'; @@ -57,6 +57,6 @@ async function handler(ctx) { item: items, }; } finally { - await browser.close(); + await context.close(); } } diff --git a/lib/routes/dailypush/tags.ts b/lib/routes/dailypush/tags.ts index c1430d29a..0cac69fc7 100644 --- a/lib/routes/dailypush/tags.ts +++ b/lib/routes/dailypush/tags.ts @@ -43,12 +43,12 @@ async function handler(ctx) { const { tag, sort = 'trending' } = ctx.req.param(); const url = `${BASE_URL}/${tag}/${sort}`; - const browser = await playwright(); + const context = await playwright(); try { - const html = await fetchPageHtml(browser, url, 'article'); + const html = await fetchPageHtml(context, url, 'article'); const $ = load(html); const list = parseArticles($, BASE_URL); - const items = await enhanceItemsWithSummaries(browser, list); + const items = await enhanceItemsWithSummaries(context, list); const pageTitle = $('title').text() || `DailyPush - ${tag.charAt(0).toUpperCase() + tag.slice(1)}`; @@ -58,6 +58,6 @@ async function handler(ctx) { item: items, }; } finally { - await browser.close(); + await context.close(); } } diff --git a/lib/routes/dailypush/utils.ts b/lib/routes/dailypush/utils.ts index 8a55e7305..134eca52f 100644 --- a/lib/routes/dailypush/utils.ts +++ b/lib/routes/dailypush/utils.ts @@ -1,11 +1,12 @@ import type { CheerioAPI } from 'cheerio'; import { load } from 'cheerio'; +import type { BrowserContext } from 'patchright'; import type { DataItem } from '@/types'; import cache from '@/utils/cache'; import logger from '@/utils/logger'; import { parseRelativeDate } from '@/utils/parse-date'; -import type { Browser, Page } from '@/utils/playwright'; +import type { Page } from '@/utils/playwright'; export const BASE_URL = 'https://www.dailypush.dev'; @@ -23,19 +24,19 @@ export interface ArticleItem { const allowedRequestTypes = new Set(['document']); async function preparePage(page: Page) { - await page.setRequestInterception(true); - page.on('request', (request) => { + await page.route('**/*', (route) => { + const request = route.request(); if (allowedRequestTypes.has(request.resourceType())) { - request.continue(); + route.continue(); return; } - request.abort(); + route.abort(); }); } -export async function fetchPageHtml(browser: Browser, url: string, waitForSelector?: string): Promise { - const page = await browser.newPage(); +export async function fetchPageHtml(context: BrowserContext, url: string, waitForSelector?: string): Promise { + const page = await context.newPage(); await preparePage(page); try { @@ -259,9 +260,9 @@ export function parseArticles($: CheerioAPI, baseUrl: string): ArticleItem[] { /** * Enhance items with full summaries from dailypush article pages. - * Uses the provided browser; opens a new tab per URL (document requests only). Caller must close the browser. + * Uses the provided context; opens a new tab per URL (document requests only). Caller must close the context. */ -export async function enhanceItemsWithSummaries(browser: Browser, items: ArticleItem[]): Promise { +export async function enhanceItemsWithSummaries(context: BrowserContext, items: ArticleItem[]): Promise { const itemsWithUrl = items.filter((item) => item.dailyPushUrl !== undefined); const itemsWithoutUrl: DataItem[] = items.filter((item) => item.dailyPushUrl === undefined); @@ -269,7 +270,7 @@ export async function enhanceItemsWithSummaries(browser: Browser, items: Article itemsWithUrl.map((item) => cache.tryGet(item.dailyPushUrl!, async () => { try { - const html = await fetchPageHtml(browser, item.dailyPushUrl!, 'p.font-ibm-plex-sans.leading-relaxed'); + const html = await fetchPageHtml(context, item.dailyPushUrl!, 'p.font-ibm-plex-sans.leading-relaxed'); const $ = load(html); const summary = $('p.font-ibm-plex-sans.leading-relaxed'); if (summary.length > 0 && summary.text().trim()) { diff --git a/lib/routes/dcard/section.ts b/lib/routes/dcard/section.ts index a32f14582..9c30bc07c 100644 --- a/lib/routes/dcard/section.ts +++ b/lib/routes/dcard/section.ts @@ -26,7 +26,7 @@ export const route: Route = { async function handler(ctx) { const { type = 'latest', section = 'posts' } = ctx.req.param(); const limit = ctx.req.query('limit') ? Number(ctx.req.query('limit')) : 30; - const browser = await playwright(); + const context = await playwright(); let link = 'https://www.dcard.tw/f'; let api = 'https://www.dcard.tw/service/api/v2'; @@ -48,10 +48,10 @@ async function handler(ctx) { title += '最新'; } - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); await page.setExtraHTTPHeaders({ referer: `https://www.dcard.tw/f/${section}`, @@ -60,7 +60,7 @@ async function handler(ctx) { await page.goto(`${api}&limit=100`); await page.waitForSelector('body > pre'); const response = await page.evaluate(() => document.querySelector('body > pre').textContent); - const cookies = await cache.tryGet('dcard:cookies', () => page.cookies(), 3600, false); + const cookies = await cache.tryGet('dcard:cookies', () => page.context().cookies(), 3600, false); await page.close(); const data = JSON.parse(response); @@ -76,8 +76,8 @@ async function handler(ctx) { })); // parse fulltext for first `limit` items - const result = await utils.ProcessFeed(items, cookies, browser, limit, cache); - await browser.close(); + const result = await utils.ProcessFeed(items, cookies, context, limit, cache); + await context.close(); return { title, diff --git a/lib/routes/dcard/utils.ts b/lib/routes/dcard/utils.ts index 35bdc8594..fffa67a36 100644 --- a/lib/routes/dcard/utils.ts +++ b/lib/routes/dcard/utils.ts @@ -1,6 +1,6 @@ import pMap from 'p-map'; -const ProcessFeed = async (items, cookies, browser, limit, cache) => { +const ProcessFeed = async (items, cookies, context, limit, cache) => { let newCookies = []; const result = await pMap( items.slice(0, limit), @@ -10,19 +10,19 @@ const ProcessFeed = async (items, cookies, browser, limit, cache) => { let response; // try catch 处理被删除的帖子 try { - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? request.continue() : request.abort(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? route.continue() : route.abort(); }); await page.setExtraHTTPHeaders({ referer: `https://www.dcard.tw/f/${i.forumAlias}/p/${i.id}`, }); - await page.setCookie(...cookies); + await page.context().addCookies(cookies); await page.goto(url); await page.waitForSelector('body > pre'); response = await page.evaluate(() => document.querySelector('body > pre').textContent); - newCookies = await page.cookies(); + newCookies = await page.context().cookies(); await page.close(); const data = JSON.parse(response); diff --git a/lib/routes/douyin/hashtag.ts b/lib/routes/douyin/hashtag.ts index 47f3f01fa..77541e05e 100644 --- a/lib/routes/douyin/hashtag.ts +++ b/lib/routes/douyin/hashtag.ts @@ -47,12 +47,12 @@ async function handler(ctx) { const tagData = await cache.tryGet( `douyin:hashtag:${cid}`, async () => { - const browser = await playwright(); - const page = await browser.newPage(); - await page.setRequestInterception(true); + const context = await playwright(); + const page = await context.newPage(); let awemeList = ''; - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? route.continue() : route.abort(); }); page.on('response', async (response) => { const request = response.request(); @@ -61,11 +61,11 @@ async function handler(ctx) { } }); await page.goto(tagUrl, { - waitUntil: 'networkidle2', + waitUntil: 'networkidle', }); await page.waitForSelector('#RENDER_DATA'); const html = await page.evaluate(() => document.querySelector('#RENDER_DATA').textContent); - await browser.close(); + await context.close(); const renderData = JSON.parse(decodeURIComponent(html)); const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail); diff --git a/lib/routes/douyin/live.ts b/lib/routes/douyin/live.ts index c1d3f7a43..93cb864a3 100644 --- a/lib/routes/douyin/live.ts +++ b/lib/routes/douyin/live.ts @@ -42,12 +42,11 @@ async function handler(ctx) { `douyin:live:${rid}`, async () => { let roomInfo; - const browser = await playwright(); - const page = await browser.newPage(); - await page.setRequestInterception(true); - - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'stylesheet' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort(); + const context = await playwright(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'stylesheet' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? route.continue() : route.abort(); }); page.on('response', async (response) => { const request = response.request(); @@ -57,9 +56,9 @@ async function handler(ctx) { }); logger.http(`Requesting ${pageUrl}`); await page.goto(pageUrl, { - waitUntil: 'networkidle2', + waitUntil: 'networkidle', }); - await browser.close(); + await context.close(); return roomInfo; }, diff --git a/lib/routes/douyin/user.ts b/lib/routes/douyin/user.ts index bbfbb8edc..ee7994e93 100644 --- a/lib/routes/douyin/user.ts +++ b/lib/routes/douyin/user.ts @@ -50,12 +50,11 @@ async function handler(ctx) { `douyin:user:${uid}`, async () => { let postData; - const browser = await playwright(); - const page = await browser.newPage(); - await page.setRequestInterception(true); - - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort(); + const context = await playwright(); + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? route.continue() : route.abort(); }); page.on('response', async (response) => { const request = response.request(); @@ -66,10 +65,10 @@ async function handler(ctx) { logger.http(`Requesting ${pageUrl}`); await page.goto(pageUrl, { - waitUntil: 'networkidle2', + waitUntil: 'networkidle', }); - await browser.close(); + await context.close(); if (!postData) { throw new Error('Empty post data. The request may be filtered by WAF.'); diff --git a/lib/routes/fortnite/news.ts b/lib/routes/fortnite/news.ts index a64faa80d..2c2fad005 100644 --- a/lib/routes/fortnite/news.ts +++ b/lib/routes/fortnite/news.ts @@ -40,14 +40,13 @@ async function handler(ctx) { const apiUrl = `https://www.fortnite.com/api/blog/getPosts?category=&postsPerPage=0&offset=0&locale=${language}&rootPageSlug=blog`; // Use Playwright instead of got, which may be blocked by anti-crawling scripts with response code 403. - const browser = await playwright(); - const page = await browser.newPage(); + const context = await playwright(); + const page = await context.newPage(); - // intercept all requests - await page.setRequestInterception(true); // only document is allowed - page.on('request', (request) => { - request.resourceType() === 'document' ? request.continue() : request.abort(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' ? route.continue() : route.abort(); }); // log manually (necessary for Playwright) @@ -55,7 +54,7 @@ async function handler(ctx) { let data; try { const response = await page.goto(apiUrl, { - waitUntil: 'networkidle0', + waitUntil: 'networkidle', }); if (!response) { throw new Error(`No response received from ${apiUrl}`); @@ -73,7 +72,7 @@ async function handler(ctx) { data = await response.json(); } finally { await page.close(); - await browser.close(); + await context.close(); } const { blogList: list } = data; diff --git a/lib/routes/gov/customs/list.ts b/lib/routes/gov/customs/list.ts index 21ca8ee68..e177ce6fc 100644 --- a/lib/routes/gov/customs/list.ts +++ b/lib/routes/gov/customs/list.ts @@ -61,12 +61,12 @@ async function handler(ctx) { break; } - const browser = await playwright(); + const context = await playwright(); const list = await cache.tryGet( link, async () => { - const response = await playwrightGet(link, browser); + const response = await playwrightGet(link, context); const $ = load(response); const list = $('[class^="conList_ul"] li') .toArray() @@ -90,7 +90,7 @@ async function handler(ctx) { if (info.link.endsWith('.pdf') || info.link.endsWith('.doc')) { return info; } - const response = await playwrightGet(info.link, browser); + const response = await playwrightGet(info.link, context); const $ = load(response); let date; @@ -110,7 +110,7 @@ async function handler(ctx) { ) ); - await browser.close(); + await context.close(); return { title: `中国海关-${channelName}`, diff --git a/lib/routes/gov/customs/utils.ts b/lib/routes/gov/customs/utils.ts index 798cdf022..9a2d92359 100644 --- a/lib/routes/gov/customs/utils.ts +++ b/lib/routes/gov/customs/utils.ts @@ -1,11 +1,11 @@ const host = 'http://www.customs.gov.cn'; -const playwrightGet = async (url, browser) => { - const page = await browser.newPage(); +const playwrightGet = async (url, context) => { + const page = await context.newPage(); await page.setExtraHTTPHeaders({ referer: host }); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + await page.route('**/*', (route) => { + const request = route.request(); + request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort(); }); await page.goto(url, { waitUntil: 'domcontentloaded', diff --git a/lib/routes/gov/hangzhou/zjzwfw.ts b/lib/routes/gov/hangzhou/zjzwfw.ts index 24a01b5c0..949c9ac27 100644 --- a/lib/routes/gov/hangzhou/zjzwfw.ts +++ b/lib/routes/gov/hangzhou/zjzwfw.ts @@ -1,20 +1,20 @@ import logger from '@/utils/logger'; -export async function crawler(item: any, browser: any): Promise { +export async function crawler(item: any, context: any): Promise { try { let response = ''; - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { + const page = await context.newPage(); + await page.route('**/*', (route) => { + const request = route.request(); const resourceType = request.resourceType(); if (['document', 'script', 'stylesheet', 'xhr'].includes(resourceType)) { - request.continue(); + route.continue(); } else { - request.abort(); + route.abort(); } }); await page.goto(item.link, { - waitUntil: 'networkidle0', + waitUntil: 'networkidle', timeout: 29000, }); const selector = '.item-left .item .title .button'; diff --git a/lib/routes/gov/hangzhou/zwfw.tsx b/lib/routes/gov/hangzhou/zwfw.tsx index 224d7e555..f5cd4cdfe 100644 --- a/lib/routes/gov/hangzhou/zwfw.tsx +++ b/lib/routes/gov/hangzhou/zwfw.tsx @@ -216,7 +216,7 @@ async function handler() { const host = 'https://www.hangzhou.gov.cn/col/col1256349/index.html'; const response = await ofetch(host); - const browser = await playwright(); + const context = await playwright(); const link = host; const formatted = response .replace('