xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: replace playwright with patchright (#21976) 

* refactor: use patchwright

* refactor: migrate setRequestInterception and waitUntil

* fix: overwrite ua

* refactor: drop puppeteer cookie/requestfinished shims

Migrate page.cookies/setCookie and browser.cookies/setCookie call sites to the native context APIs (page.context().cookies/addCookies), rewrite requestfinished handlers to await request.response() instead of consuming the synchronous puppeteer-shaped payload, and remove the now-redundant browser.close context-cleanup wrapper. Playwright already handles all of these natively; the shims existed only for the puppeteer -> playwright migration.

* refactor: remove page.authenticate, page.setUserAgent,  browser.newPage(), browser.userAgent() shims

update cookie utility tests

* refactor: replace browser instance with context in Playwright usage across multiple routes

- Updated all instances of `browser` to `context` in Playwright-related functions for consistency and improved resource management.
- Ensured proper closure of context after operations to prevent memory leaks.
- Adjusted request interception methods to utilize the new context variable.

* fix: direct type import

* refactor: remove Puppeteer compatibility layers and unify page handling with Playwright

* fix: test case

* chore: bump patchright to v1.60.0

* chore: pin patchright to 1.59.1

nft can't resolve browser.json due to https://github.com/microsoft/playwright/pull/40093

* chore: pin to node 24.15 for playwright

browser install on node 24.16 stucks due to https://github.com/microsoft/playwright/issues/40724

* fix: update patchright and patchright-core to version 1.60.2 and adjust minify-docker.js for compatibility

* fix: import BrowserContext from patchright
This commit is contained in:
Tony 2026-06-07 02:43:20 +08:00 committed by GitHub
parent 6886265f1b
commit cfcdc34e1f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
104 changed files with 730 additions and 1111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/issue-command.yml vendored
View File

@ -120,7 +120,7 @@ jobs:
cache: 'pnpm'
- name: Install dependencies (pnpm)
run: pnpm i && pnpm rb && pnpm exec playwright install chromium
run: pnpm i && pnpm rb && pnpm exec patchright install chromium
- name: Fetch affected routes
id: fetch-route

4
.github/workflows/test.yml vendored
View File

@ -39,7 +39,7 @@ jobs:
- name: Install dependencies (pnpm)
run: pnpm i
- name: Run postinstall script for dependencies
run: pnpm rb && pnpm exec playwright install chromium
run: pnpm rb && pnpm exec patchright install chromium
- name: Build routes
run: pnpm build
- name: Build worker routes
@ -90,7 +90,7 @@ jobs:
run: pnpm build
- name: Install bundled Chromium
if: ${{ matrix.chromium.dependency == '' }}
run: pnpm exec playwright install chromium
run: pnpm exec patchright install chromium
- name: Install Chromium
if: ${{ matrix.chromium.dependency != '' }}
# 'chromium-browser' from Ubuntu APT repo is a dummy package. Its version (85.0.4183.83) means

10
Dockerfile
View File

@ -40,7 +40,7 @@ WORKDIR /ver
COPY ./package.json /app/
RUN \
set -ex && \
grep -Po '(?<="playwright": ")[^\s"]*(?=")' /app/package.json | tee /ver/.playwright_version && \
grep -Po '(?<="patchright": ")[^\s"]*(?=")' /app/package.json | tee /ver/.patchright_version && \
grep -Po '(?<="@vercel/nft": ")[^\s"]*(?=")' /app/package.json | tee /ver/.nft_version && \
grep -Po '(?<="fs-extra": ")[^\s"]*(?=")' /app/package.json | tee /ver/.fs_extra_version
@ -88,12 +88,12 @@ FROM node:24-bookworm-slim AS chromium-downloader
# Yeah, downloading Chromium never needs those dependencies below.
WORKDIR /app
COPY --from=dep-version-parser /ver/.playwright_version /app/.playwright_version
COPY --from=dep-version-parser /ver/.patchright_version /app/.patchright_version
ARG TARGETPLATFORM
ARG USE_CHINA_NPM_REGISTRY=0
ARG PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
# The official recommended way to use Playwright on x86(_64) is to use the bundled browser.
# The official recommended way to use Patchright on x86(_64) is to use the bundled browser.
RUN \
set -ex ; \
if [ "$PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD" = 0 ] && [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \
@ -106,9 +106,9 @@ RUN \
unset PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD && \
export PLAYWRIGHT_BROWSERS_PATH=/app/node_modules/.cache/ms-playwright && \
corepack enable pnpm && \
pnpm --allow-build=playwright add playwright@$(cat /app/.playwright_version) --save-prod && \
pnpm --allow-build=patchright --allow-build=patchright-core add patchright@$(cat /app/.patchright_version) --save-prod && \
pnpm rb && \
pnpm exec playwright install chromium ; \
pnpm exec patchright install chromium ; \
else \
mkdir -p /app/node_modules/.cache/ms-playwright ; \
fi;

11
lib/bilibili-video-route.test.ts
View File

@ -11,15 +11,13 @@ const destroy = vi.fn();
const getPlaywrightPage = vi.fn();
const goto = vi.fn();
const on = vi.fn();
const setCookie = vi.fn();
const setRequestInterception = vi.fn();
const pageRoute = vi.fn();
const waitForResponse = vi.fn();
const page = {
goto,
on,
setCookie,
setRequestInterception,
route: pageRoute,
waitForResponse,
};
@ -66,8 +64,7 @@ describe('/bilibili/user/video/:uid', () => {
getPlaywrightPage.mockReset();
goto.mockReset();
on.mockReset();
setCookie.mockReset();
setRequestInterception.mockReset();
pageRoute.mockReset();
waitForResponse.mockReset();
});
@ -109,6 +106,7 @@ describe('/bilibili/user/video/:uid', () => {
getPlaywrightPage.mockImplementation(async (_url, options) => {
await options.onBeforeLoad?.(page);
return {
context: {},
destroy,
page,
};
@ -171,6 +169,7 @@ describe('/bilibili/user/video/:uid', () => {
getPlaywrightPage.mockImplementation(async (_url, options) => {
await options.onBeforeLoad?.(page);
return {
context: {},
destroy,
page,
};

12
lib/routes/acs/journal.tsx
View File

@ -28,14 +28,14 @@ async function handler(ctx) {
let title = '';
const browser = await playwright();
const context = await playwright();
const items = await cache.tryGet(
currentUrl,
async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(currentUrl, {
waitUntil: 'domcontentloaded',
@ -76,7 +76,7 @@ async function handler(ctx) {
false
);
await browser.close();
await context.close();
return {
title,

6
lib/routes/aip/journal-pupp.ts
View File

@ -18,12 +18,12 @@ const handler = async (ctx) => {
}
// use Playwright due to the obstacle by cloudflare challenge
const browser = await playwright();
const context = await playwright();
const { jrnlName, list } = await cache.tryGet(
jrnlUrl,
async () => {
const response = await playwrightGet(jrnlUrl, browser);
const response = await playwrightGet(jrnlUrl, context);
const $ = load(response);
const jrnlName = $('.header-journal-title').text();
const list = $('.card')
@ -52,7 +52,7 @@ const handler = async (ctx) => {
false
);
await browser.close();
await context.close();
return {
title: jrnlName,

10
lib/routes/aip/utils.tsx
View File

@ -1,11 +1,11 @@
import { renderToString } from 'hono/jsx/dom/server';
const playwrightGet = async (url, browser) => {
const page = await browser.newPage();
const playwrightGet = async (url, context) => {
const page = await context.newPage();
// await page.setExtraHTTPHeaders({ referer: host });
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',

12
lib/routes/alternativeto/utils.ts
View File

@ -4,17 +4,17 @@ const baseURL = 'https://alternativeto.net';
const playwrightGet = (url, cache) =>
cache.tryGet(url, async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',
});
const html = await page.evaluate(() => document.documentElement.innerHTML);
await browser.close();
await context.close();
return html;
});

12
lib/routes/apkpure/versions.ts
View File

@ -28,11 +28,11 @@ async function handler(ctx) {
const baseUrl = 'https://apkpure.com';
const link = `${baseUrl}/${region}/${pkg}/versions`;
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
logger.http(`Requesting ${link}`);
await page.goto(link, {
@ -40,7 +40,7 @@ async function handler(ctx) {
});
const r = await page.evaluate(() => document.documentElement.innerHTML);
await browser.close();
await context.close();
const $ = load(r);
const img = new URL($('.ver-top img').attr('src'));

2
lib/routes/bilibili/cache.ts
View File

@ -51,7 +51,7 @@ const getCookie = (disableConfig = false) => {
waitForRequest = new Promise<string>((resolve) => {
page.on('requestfinished', async (request) => {
if (request.url() === 'https://api.bilibili.com/x/web-interface/nav') {
const cookies = await page.cookies();
const cookies = await page.context().cookies();
let cookieString = cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ');
cookieString = cookieString.replace(/b_lsid=[0-9A-F]+_[0-9A-F]+/, `b_lsid=${utils.lsid()}`);
resolve(cookieString);

8
lib/routes/bilibili/video.ts
View File

@ -140,7 +140,7 @@ async function applyCookie(page: Page, cookie: string) {
.filter((item) => item !== undefined);
if (cookies.length > 0) {
await page.setCookie(...cookies);
await page.context().addCookies(cookies);
}
}
@ -184,9 +184,9 @@ async function fetchVideoListFromBrowser(uid: string): Promise<VideoListData> {
await applyCookie(page, cookie);
}
await page.setRequestInterception(true);
page.on('request', (request) => {
allowedBrowserRequestTypes.has(request.resourceType()) ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
allowedBrowserRequestTypes.has(request.resourceType()) ? route.continue() : route.abort();
});
},
gotoConfig: { waitUntil: 'domcontentloaded' },

4
lib/routes/bilibili/wasm-exec.ts
View File

@ -2,7 +2,7 @@
// oxlint-disable no-unused-vars
/* eslint-disable prefer-rest-params */
/* eslint-disable default-case */
/* eslint-disable unicorn/consistent-function-scoping */
// oxlint-disable unicorn/consistent-function-scoping
/* eslint-disable no-console */
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
@ -634,7 +634,7 @@
_makeFuncWrapper(id) {
// somehow avoiding aliasing this with an arrow function doesn't work
// eslint-disable-next-line unicorn/no-this-assignment, @typescript-eslint/no-this-alias
// oxlint-disable-next-line unicorn/no-this-assignment typescript/no-this-alias
const go = this;
return function () {
const event = { id, this: this, args: arguments };

20
lib/routes/bluestacks/release.ts
View File

@ -32,11 +32,11 @@ export const route: Route = {
};
async function handler() {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(pageUrl, {
waitUntil: 'domcontentloaded',
@ -59,10 +59,10 @@ async function handler() {
await Promise.all(
items.map((item) =>
cache.tryGet(item.link, async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(item.link, {
waitUntil: 'domcontentloaded',
@ -79,7 +79,7 @@ async function handler() {
)
);
await browser.close();
await context.close();
return {
title: $('.article__title').text().trim(),

12
lib/routes/ccac/news.ts
View File

@ -32,21 +32,21 @@ export const route: Route = {
};
async function handler(ctx) {
const browser = await playwright();
const context = await playwright();
const lang = ctx.req.param('lang') ?? 'sc';
const type = utils.TYPE[ctx.req.param('type')];
const BASE = utils.langBase(lang);
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(BASE, {
waitUntil: 'domcontentloaded',
});
const articles = await page.evaluate(() => window.articles);
await browser.close();
await context.close();
const list = utils
.typeFilter(articles, type)

12
lib/routes/chinadegrees/province.tsx
View File

@ -82,11 +82,11 @@ async function handler(ctx) {
const data = await cache.tryGet(
url,
async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',
@ -94,7 +94,7 @@ async function handler(ctx) {
await page.waitForSelector('.datalist');
const html = await page.evaluate(() => document.documentElement.innerHTML);
await browser.close();
await context.close();
const $ = load(html);
return {

12
lib/routes/chinatimes/index.ts
View File

@ -43,7 +43,7 @@ async function handler(ctx) {
const response = await ofetch(link);
const $ = load(response);
const browser = await playwright();
const context = await playwright();
const list = $('.articlebox-compact')
.toArray()
@ -66,10 +66,10 @@ async function handler(ctx) {
const items = await Promise.all(
list.map((item) =>
cache.tryGet(item.link, async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
logger.http(`Requesting ${item.link}`);
await page.goto(item.link, {
@ -98,7 +98,7 @@ async function handler(ctx) {
)
);
await browser.close();
await context.close();
return {
title: $('head title').text(),

13
lib/routes/cjlu/yjsy/index.ts
View File

@ -86,19 +86,18 @@ async function handler(ctx) {
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 10;
const url = `${host}index/${cate}.htm`;
const { page, destroy, browser } = await getPlaywrightPage(url, {
const { page, destroy } = await getPlaywrightPage(url, {
onBeforeLoad: async (page) => {
await page.setExtraHTTPHeaders(headers);
await page.setUserAgent(headers['User-Agent']);
await page.setRequestInterception(true);
page.on('request', (request) => {
allowedResourceTypes.has(request.resourceType()) ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
allowedResourceTypes.has(request.resourceType()) ? route.continue() : route.abort();
});
},
gotoConfig: { waitUntil: 'networkidle2' },
gotoConfig: { waitUntil: 'networkidle' },
});
const cookies = await browser.cookies();
const cookies = await page.context().cookies();
const cookieString = cookies.map((c) => `${c.name}=${c.value}`).join('; ');
const response = await page.content();

20
lib/routes/cmde/index.ts
View File

@ -18,12 +18,12 @@ export const route: Route = {
async function handler(ctx) {
const cate = ctx.req.param('cate') ?? 'xwdt/zxyw';
const url = `${rootURL}/${cate}/`;
const browser = await playwright();
const context = await playwright();
const data = await cache.tryGet(url, async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',
@ -52,10 +52,10 @@ async function handler(ctx) {
const items = await Promise.all(
data.items.map((item) =>
cache.tryGet(item.link, async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(item.link, {
waitUntil: 'domcontentloaded',
@ -72,7 +72,7 @@ async function handler(ctx) {
)
);
await browser.close();
await context.close();
return {
title: data.title,

13
lib/routes/colamanga/manga.ts
View File

@ -43,14 +43,13 @@ async function handler(ctx: Context) {
const id = ctx.req.param('id');
const url = `https://${domain}/${id}`;
const browser = await playwright();
const context = await playwright();
const page = await browser.newPage();
const page = await context.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
logger.http(`Requesting ${url}`);
@ -60,7 +59,7 @@ async function handler(ctx: Context) {
});
const response = await page.content();
await browser.close();
await context.close();
const $ = load(response);

6
lib/routes/cw/author.ts
View File

@ -27,11 +27,11 @@ export const route: Route = {
};
async function handler(ctx) {
const browser = await playwright();
const context = await playwright();
const { $, items } = await parsePage('author', browser, ctx);
const { $, items } = await parsePage('author', context, ctx);
await browser.close();
await context.close();
return {
title: $('head title').text(),

6
lib/routes/cw/master.ts
View File

@ -37,11 +37,11 @@ export const route: Route = {
};
async function handler(ctx) {
const browser = await playwright();
const context = await playwright();
const { $, items } = await parsePage('master', browser, ctx);
const { $, items } = await parsePage('master', context, ctx);
await browser.close();
await context.close();
return {
title: $('head title').text(),

6
lib/routes/cw/sub.ts
View File

@ -22,11 +22,11 @@ export const route: Route = {
};
async function handler(ctx) {
const browser = await playwright();
const context = await playwright();
const { $, items } = await parsePage('sub', browser, ctx);
const { $, items } = await parsePage('sub', context, ctx);
await browser.close();
await context.close();
return {
title: $('head title').text(),

6
lib/routes/cw/today.ts
View File

@ -28,11 +28,11 @@ export const route: Route = {
};
async function handler(ctx) {
const browser = await playwright();
const context = await playwright();
const { $, items } = await parsePage('today', browser, ctx);
const { $, items } = await parsePage('today', context, ctx);
await browser.close();
await context.close();
return {
title: $('head title').text(),

31
lib/routes/cw/utils.ts
View File

@ -1,5 +1,6 @@
import { load } from 'cheerio';
import { config } from '@/config';
import cache from '@/utils/cache';
import logger from '@/utils/logger';
import ofetch from '@/utils/ofetch';
@ -29,13 +30,13 @@ const pathMap = {
},
};
const getCookie = async (browser, tryGet) => {
const getCookie = async (context, tryGet) => {
if (!cookie) {
cookie = await tryGet('cw:cookie', async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
logger.http(`Requesting ${baseUrl}/user/get/cookie-bar`);
await page.goto(`${baseUrl}/user/get/cookie-bar`, {
@ -49,14 +50,14 @@ const getCookie = async (browser, tryGet) => {
return cookie;
};
const parsePage = async (path, browser, ctx) => {
const parsePage = async (path, context, ctx) => {
const pageUrl = `${baseUrl}${pathMap[path].pageUrl(ctx.req.param('channel'))}`;
const cookie = await getCookie(browser, cache.tryGet);
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const cookie = await getCookie(context, cache.tryGet);
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await setCookies(page, cookie, 'cw.com.tw');
logger.http(`Requesting ${pageUrl}`);
@ -70,7 +71,7 @@ const parsePage = async (path, browser, ctx) => {
const $ = load(response);
const list = parseList($, ctx.req.query('limit') ? Number(ctx.req.query('limit')) : pathMap[path].limit);
const items = await parseItems(list, browser, cache.tryGet);
const items = await parseItems(list, context, cache.tryGet);
return { $, items };
};
@ -88,14 +89,14 @@ const parseList = ($, limit) =>
})
.slice(0, limit);
const parseItems = (list, browser, tryGet) =>
const parseItems = (list, context, tryGet) =>
Promise.all(
list.map((item) =>
tryGet(item.link, async () => {
const response = await ofetch(item.link, {
headers: {
Cookie: await getCookie(browser, tryGet),
'User-Agent': browser.userAgent(),
Cookie: await getCookie(context, tryGet),
'User-Agent': config.ua,
},
});
const $ = load(response);

8
lib/routes/dailypush/all.ts
View File

@ -42,12 +42,12 @@ async function handler(ctx) {
const { sort = '' } = ctx.req.param();
const url = sort ? `${BASE_URL}/${sort}` : BASE_URL;
const browser = await playwright();
const context = await playwright();
try {
const html = await fetchPageHtml(browser, url, 'article');
const html = await fetchPageHtml(context, url, 'article');
const $ = load(html);
const list = parseArticles($, BASE_URL);
const items = await enhanceItemsWithSummaries(browser, list);
const items = await enhanceItemsWithSummaries(context, list);
const pageTitle = $('title').text() || 'DailyPush - All';
@ -57,6 +57,6 @@ async function handler(ctx) {
item: items,
};
} finally {
await browser.close();
await context.close();
}
}

8
lib/routes/dailypush/tags.ts
View File

@ -43,12 +43,12 @@ async function handler(ctx) {
const { tag, sort = 'trending' } = ctx.req.param();
const url = `${BASE_URL}/${tag}/${sort}`;
const browser = await playwright();
const context = await playwright();
try {
const html = await fetchPageHtml(browser, url, 'article');
const html = await fetchPageHtml(context, url, 'article');
const $ = load(html);
const list = parseArticles($, BASE_URL);
const items = await enhanceItemsWithSummaries(browser, list);
const items = await enhanceItemsWithSummaries(context, list);
const pageTitle = $('title').text() || `DailyPush - ${tag.charAt(0).toUpperCase() + tag.slice(1)}`;
@ -58,6 +58,6 @@ async function handler(ctx) {
item: items,
};
} finally {
await browser.close();
await context.close();
}
}

21
lib/routes/dailypush/utils.ts
View File

@ -1,11 +1,12 @@
import type { CheerioAPI } from 'cheerio';
import { load } from 'cheerio';
import type { BrowserContext } from 'patchright';
import type { DataItem } from '@/types';
import cache from '@/utils/cache';
import logger from '@/utils/logger';
import { parseRelativeDate } from '@/utils/parse-date';
import type { Browser, Page } from '@/utils/playwright';
import type { Page } from '@/utils/playwright';
export const BASE_URL = 'https://www.dailypush.dev';
@ -23,19 +24,19 @@ export interface ArticleItem {
const allowedRequestTypes = new Set(['document']);
async function preparePage(page: Page) {
await page.setRequestInterception(true);
page.on('request', (request) => {
await page.route('**/*', (route) => {
const request = route.request();
if (allowedRequestTypes.has(request.resourceType())) {
request.continue();
route.continue();
return;
}
request.abort();
route.abort();
});
}
export async function fetchPageHtml(browser: Browser, url: string, waitForSelector?: string): Promise<string> {
const page = await browser.newPage();
export async function fetchPageHtml(context: BrowserContext, url: string, waitForSelector?: string): Promise<string> {
const page = await context.newPage();
await preparePage(page);
try {
@ -259,9 +260,9 @@ export function parseArticles($: CheerioAPI, baseUrl: string): ArticleItem[] {
/**
* Enhance items with full summaries from dailypush article pages.
* Uses the provided browser; opens a new tab per URL (document requests only). Caller must close the browser.
* Uses the provided context; opens a new tab per URL (document requests only). Caller must close the context.
*/
export async function enhanceItemsWithSummaries(browser: Browser, items: ArticleItem[]): Promise<DataItem[]> {
export async function enhanceItemsWithSummaries(context: BrowserContext, items: ArticleItem[]): Promise<DataItem[]> {
const itemsWithUrl = items.filter((item) => item.dailyPushUrl !== undefined);
const itemsWithoutUrl: DataItem[] = items.filter((item) => item.dailyPushUrl === undefined);
@ -269,7 +270,7 @@ export async function enhanceItemsWithSummaries(browser: Browser, items: Article
itemsWithUrl.map((item) =>
cache.tryGet(item.dailyPushUrl!, async () => {
try {
const html = await fetchPageHtml(browser, item.dailyPushUrl!, 'p.font-ibm-plex-sans.leading-relaxed');
const html = await fetchPageHtml(context, item.dailyPushUrl!, 'p.font-ibm-plex-sans.leading-relaxed');
const $ = load(html);
const summary = $('p.font-ibm-plex-sans.leading-relaxed');
if (summary.length > 0 && summary.text().trim()) {

16
lib/routes/dcard/section.ts
View File

@ -26,7 +26,7 @@ export const route: Route = {
async function handler(ctx) {
const { type = 'latest', section = 'posts' } = ctx.req.param();
const limit = ctx.req.query('limit') ? Number(ctx.req.query('limit')) : 30;
const browser = await playwright();
const context = await playwright();
let link = 'https://www.dcard.tw/f';
let api = 'https://www.dcard.tw/service/api/v2';
@ -48,10 +48,10 @@ async function handler(ctx) {
title += '最新';
}
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.setExtraHTTPHeaders({
referer: `https://www.dcard.tw/f/${section}`,
@ -60,7 +60,7 @@ async function handler(ctx) {
await page.goto(`${api}&limit=100`);
await page.waitForSelector('body > pre');
const response = await page.evaluate(() => document.querySelector('body > pre').textContent);
const cookies = await cache.tryGet('dcard:cookies', () => page.cookies(), 3600, false);
const cookies = await cache.tryGet('dcard:cookies', () => page.context().cookies(), 3600, false);
await page.close();
const data = JSON.parse(response);
@ -76,8 +76,8 @@ async function handler(ctx) {
}));
// parse fulltext for first `limit` items
const result = await utils.ProcessFeed(items, cookies, browser, limit, cache);
await browser.close();
const result = await utils.ProcessFeed(items, cookies, context, limit, cache);
await context.close();
return {
title,

14
lib/routes/dcard/utils.ts
View File

@ -1,6 +1,6 @@
import pMap from 'p-map';
const ProcessFeed = async (items, cookies, browser, limit, cache) => {
const ProcessFeed = async (items, cookies, context, limit, cache) => {
let newCookies = [];
const result = await pMap(
items.slice(0, limit),
@ -10,19 +10,19 @@ const ProcessFeed = async (items, cookies, browser, limit, cache) => {
let response;
// try catch 处理被删除的帖子
try {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? route.continue() : route.abort();
});
await page.setExtraHTTPHeaders({
referer: `https://www.dcard.tw/f/${i.forumAlias}/p/${i.id}`,
});
await page.setCookie(...cookies);
await page.context().addCookies(cookies);
await page.goto(url);
await page.waitForSelector('body > pre');
response = await page.evaluate(() => document.querySelector('body > pre').textContent);
newCookies = await page.cookies();
newCookies = await page.context().cookies();
await page.close();
const data = JSON.parse(response);

14
lib/routes/douyin/hashtag.ts
View File

@ -47,12 +47,12 @@ async function handler(ctx) {
const tagData = await cache.tryGet(
`douyin:hashtag:${cid}`,
async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
const context = await playwright();
const page = await context.newPage();
let awemeList = '';
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? route.continue() : route.abort();
});
page.on('response', async (response) => {
const request = response.request();
@ -61,11 +61,11 @@ async function handler(ctx) {
}
});
await page.goto(tagUrl, {
waitUntil: 'networkidle2',
waitUntil: 'networkidle',
});
await page.waitForSelector('#RENDER_DATA');
const html = await page.evaluate(() => document.querySelector('#RENDER_DATA').textContent);
await browser.close();
await context.close();
const renderData = JSON.parse(decodeURIComponent(html));
const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail);

15
lib/routes/douyin/live.ts
View File

@ -42,12 +42,11 @@ async function handler(ctx) {
`douyin:live:${rid}`,
async () => {
let roomInfo;
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'stylesheet' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'stylesheet' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? route.continue() : route.abort();
});
page.on('response', async (response) => {
const request = response.request();
@ -57,9 +56,9 @@ async function handler(ctx) {
});
logger.http(`Requesting ${pageUrl}`);
await page.goto(pageUrl, {
waitUntil: 'networkidle2',
waitUntil: 'networkidle',
});
await browser.close();
await context.close();
return roomInfo;
},

15
lib/routes/douyin/user.ts
View File

@ -50,12 +50,11 @@ async function handler(ctx) {
`douyin:user:${uid}`,
async () => {
let postData;
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? route.continue() : route.abort();
});
page.on('response', async (response) => {
const request = response.request();
@ -66,10 +65,10 @@ async function handler(ctx) {
logger.http(`Requesting ${pageUrl}`);
await page.goto(pageUrl, {
waitUntil: 'networkidle2',
waitUntil: 'networkidle',
});
await browser.close();
await context.close();
if (!postData) {
throw new Error('Empty post data. The request may be filtered by WAF.');

15
lib/routes/fortnite/news.ts
View File

@ -40,14 +40,13 @@ async function handler(ctx) {
const apiUrl = `https://www.fortnite.com/api/blog/getPosts?category=&postsPerPage=0&offset=0&locale=${language}&rootPageSlug=blog`;
// Use Playwright instead of got, which may be blocked by anti-crawling scripts with response code 403.
const browser = await playwright();
const page = await browser.newPage();
const context = await playwright();
const page = await context.newPage();
// intercept all requests
await page.setRequestInterception(true);
// only document is allowed
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
// log manually (necessary for Playwright)
@ -55,7 +54,7 @@ async function handler(ctx) {
let data;
try {
const response = await page.goto(apiUrl, {
waitUntil: 'networkidle0',
waitUntil: 'networkidle',
});
if (!response) {
throw new Error(`No response received from ${apiUrl}`);
@ -73,7 +72,7 @@ async function handler(ctx) {
data = await response.json();
} finally {
await page.close();
await browser.close();
await context.close();
}
const { blogList: list } = data;

8
lib/routes/gov/customs/list.ts
View File

@ -61,12 +61,12 @@ async function handler(ctx) {
break;
}
const browser = await playwright();
const context = await playwright();
const list = await cache.tryGet(
link,
async () => {
const response = await playwrightGet(link, browser);
const response = await playwrightGet(link, context);
const $ = load(response);
const list = $('[class^="conList_ul"] li')
.toArray()
@ -90,7 +90,7 @@ async function handler(ctx) {
if (info.link.endsWith('.pdf') || info.link.endsWith('.doc')) {
return info;
}
const response = await playwrightGet(info.link, browser);
const response = await playwrightGet(info.link, context);
const $ = load(response);
let date;
@ -110,7 +110,7 @@ async function handler(ctx) {
)
);
await browser.close();
await context.close();
return {
title: `中国海关-${channelName}`,

10
lib/routes/gov/customs/utils.ts
View File

@ -1,11 +1,11 @@
const host = 'http://www.customs.gov.cn';
const playwrightGet = async (url, browser) => {
const page = await browser.newPage();
const playwrightGet = async (url, context) => {
const page = await context.newPage();
await page.setExtraHTTPHeaders({ referer: host });
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',

14
lib/routes/gov/hangzhou/zjzwfw.ts
View File

@ -1,20 +1,20 @@
import logger from '@/utils/logger';
export async function crawler(item: any, browser: any): Promise<string> {
export async function crawler(item: any, context: any): Promise<string> {
try {
let response = '';
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
const resourceType = request.resourceType();
if (['document', 'script', 'stylesheet', 'xhr'].includes(resourceType)) {
request.continue();
route.continue();
} else {
request.abort();
route.abort();
}
});
await page.goto(item.link, {
waitUntil: 'networkidle0',
waitUntil: 'networkidle',
timeout: 29000,
});
const selector = '.item-left .item .title .button';

6
lib/routes/gov/hangzhou/zwfw.tsx
View File

@ -216,7 +216,7 @@ async function handler() {
const host = 'https://www.hangzhou.gov.cn/col/col1256349/index.html';
const response = await ofetch(host);
const browser = await playwright();
const context = await playwright();
const link = host;
const formatted = response
.replace('<script type="text/xml">', '')
@ -251,7 +251,7 @@ async function handler() {
const host = new URL(item.link).hostname;
if (host === 'www.zjzwfw.gov.cn') {
// 来源为浙江政务服务网
const content = await crawler(item, browser);
const content = await crawler(item, context);
const $ = load(content);
item.description = renderDescription(analyzer($('.item-left .item .bg_box')));
item.author = '浙江政务服务网';
@ -278,7 +278,7 @@ async function handler() {
)
);
await browser.close();
await context.close();
return {
allowEmpty: true,
title: '杭州市人民政府-政务服务公开',

20
lib/routes/gov/pbc/goutongjiaoliu.ts
View File

@ -33,11 +33,11 @@ export const route: Route = {
async function handler() {
const link = 'http://www.pbc.gov.cn/goutongjiaoliu/113456/113469/index.html';
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(link, {
waitUntil: 'domcontentloaded',
@ -59,10 +59,10 @@ async function handler() {
const items = await Promise.all(
list.map((item) =>
cache.tryGet(item.link, async () => {
const detailPage = await browser.newPage();
await detailPage.setRequestInterception(true);
detailPage.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const detailPage = await context.newPage();
await detailPage.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await detailPage.goto(item.link, {
waitUntil: 'domcontentloaded',
@ -76,7 +76,7 @@ async function handler() {
)
);
await browser.close();
await context.close();
return {
title: '中国人民银行 - 沟通交流',

20
lib/routes/gov/pbc/trade-announcement.ts
View File

@ -27,11 +27,11 @@ export const route: Route = {
async function handler() {
const link = 'http://www.pbc.gov.cn/zhengcehuobisi/125207/125213/125431/125475/index.html';
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(link, {
waitUntil: 'domcontentloaded',
@ -52,10 +52,10 @@ async function handler() {
const items = await Promise.all(
list.map((item) =>
cache.tryGet(item.link, async () => {
const detailPage = await browser.newPage();
await detailPage.setRequestInterception(true);
detailPage.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const detailPage = await context.newPage();
await detailPage.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await detailPage.goto(item.link, {
waitUntil: 'domcontentloaded',
@ -69,7 +69,7 @@ async function handler() {
)
);
await browser.close();
await context.close();
return {
title: '中国人民银行 - 货币政策司公开市场交易公告',

13
lib/routes/hitcon/zeroday.tsx
View File

@ -46,12 +46,11 @@ async function handler(ctx: Context): Promise<Data> {
url += `/${status}`;
}
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
logger.http(`Requesting ${url}`);
@ -60,7 +59,7 @@ async function handler(ctx: Context): Promise<Data> {
});
const response = await page.evaluate(() => document.documentElement.innerHTML);
await browser.close();
await context.close();
const $ = load(response);
const items: DataItem[] = $('.zdui-strip-list>li')

12
lib/routes/hkushop/vinyl-or-picture-lp.ts
View File

@ -41,11 +41,11 @@ async function handler(ctx) {
const cat = ctx.req.param('cat') ?? '';
const url = cat ? `${baseUrl}/vinyl-or-picture-lp.html?cat=${cat}` : `${baseUrl}/vinyl-or-picture-lp.html`;
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',
@ -53,7 +53,7 @@ async function handler(ctx) {
const response = await page.content();
await page.close();
await browser.close();
await context.close();
const $ = load(response);

13
lib/routes/hottoys/index.ts
View File

@ -25,15 +25,14 @@ async function handler() {
const baseUrl = 'https://www.hottoys.com.hk';
// 导入 Playwright 工具类并初始化浏览器实例
const browser = await playwright();
const context = await playwright();
// 打开一个新标签页
const page = await browser.newPage();
const page = await context.newPage();
// 拦截所有请求
await page.setRequestInterception(true);
page.on('request', (request) => {
await page.route('**/*', (route) => {
const request = route.request();
// 在这次例子，我们只允许 HTML 请求
request.resourceType() === 'document' ? request.continue() : request.abort();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await page.goto(baseUrl, {
@ -55,7 +54,7 @@ async function handler() {
guid: a.attr('href'),
};
});
await browser.close();
await context.close();
return {
title: 'Hot Toys New Products',
link: baseUrl,

12
lib/routes/ielts/index.ts
View File

@ -28,11 +28,11 @@ async function handler() {
const html = await cache.tryGet(
targetUrl,
async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(targetUrl, {
waitUntil: 'domcontentloaded',
@ -40,7 +40,7 @@ async function handler() {
await page.waitForSelector('div.container');
const html = await page.evaluate(() => document.documentElement.innerHTML);
await browser.close();
await context.close();
return html;
},
config.cache.routeExpire,

12
lib/routes/iqiyi/video.ts
View File

@ -37,14 +37,14 @@ async function handler(ctx) {
const link = `https://www.iqiyi.com/u/${uid}/videos`;
// Use Playwright because iqiyi page has a delay.
const browser = await playwright();
const context = await playwright();
const data = await cache.tryGet(
link,
async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
logger.http(`Requesting ${link}`);
await page.goto(link, {
@ -70,7 +70,7 @@ async function handler(ctx) {
config.cache.routeExpire,
false
);
await browser.close();
await context.close();
return data;
}

8
lib/routes/iwara/ranking.ts
View File

@ -60,13 +60,13 @@ async function handler(ctx) {
async () => {
const { page, destroy } = await getPlaywrightPage(url, {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' || request.resourceType() === 'fetch' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' || request.resourceType() === 'fetch' ? route.continue() : route.abort();
});
},
gotoConfig: {
waitUntil: 'networkidle0',
waitUntil: 'networkidle',
},
});

30
lib/routes/javdb/utils.ts
View File

@ -18,18 +18,20 @@ const ProcessItems = async (ctx, currentUrl, title) => {
const rootUrl = `https://${domain}`;
const { page, destroy, browser } = await getPlaywrightPage('about:blank');
const { page, destroy, context } = await getPlaywrightPage('about:blank');
if (config.javdb.session) {
await browser.setCookie({
name: '_jdb_session',
value: config.javdb.session,
domain,
path: '/',
});
await page.context().addCookies([
{
name: '_jdb_session',
value: config.javdb.session,
domain,
path: '/',
},
]);
}
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await page.goto(url.href, {
waitUntil: 'domcontentloaded',
@ -56,10 +58,10 @@ const ProcessItems = async (ctx, currentUrl, title) => {
items = await Promise.all(
items.map((item) =>
cache.tryGet(item.link, async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
logger.http(`Requesting ${item.link}`);
await page.goto(item.link, {

8
lib/routes/javtrailers/casts.ts
View File

@ -27,14 +27,14 @@ export const route: Route = {
async function handler(ctx) {
const { cast } = ctx.req.param();
const browser = await playwright();
const response = await playwrightFetch(`${baseUrl}/api/casts/${cast}?page=0`, browser);
const context = await playwright();
const response = await playwrightFetch(`${baseUrl}/api/casts/${cast}?page=0`, context);
const list = parseList(response.videos);
const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => getItem(item, browser))));
const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => getItem(item, context))));
await browser.close();
await context.close();
return {
title: `Watch ${response.cast.name} Jav Online | Japanese Adult Video - JavTrailers.com`,

8
lib/routes/javtrailers/categories.ts
View File

@ -27,14 +27,14 @@ export const route: Route = {
async function handler(ctx) {
const { category } = ctx.req.param();
const browser = await playwright();
const response = await playwrightFetch(`${baseUrl}/api/categories/${category}?page=0`, browser);
const context = await playwright();
const response = await playwrightFetch(`${baseUrl}/api/categories/${category}?page=0`, context);
const list = parseList(response.videos);
const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => getItem(item, browser))));
const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => getItem(item, context))));
await browser.close();
await context.close();
return {
title: `Watch ${response.category.name} Jav Online | Japanese Adult Video - JavTrailers.com`,

8
lib/routes/javtrailers/studios.ts
View File

@ -26,14 +26,14 @@ export const route: Route = {
async function handler(ctx) {
const { studio } = ctx.req.param();
const browser = await playwright();
const response = await playwrightFetch(`${baseUrl}/api/studios/${studio}?page=0`, browser);
const context = await playwright();
const response = await playwrightFetch(`${baseUrl}/api/studios/${studio}?page=0`, context);
const list = parseList(response.videos);
const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => getItem(item, browser))));
const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => getItem(item, context))));
await browser.close();
await context.close();
return {
title: `${response.studio.hotDvdIds?.join(' ') ?? response.studio.name} Jav Online | Japanese Adult Video - JavTrailers.com`,

14
lib/routes/javtrailers/utils.ts
View File

@ -27,12 +27,12 @@ export const parseList = (videos) =>
contentId: item.contentId,
}));
export const playwrightFetch = async (url: string, browser) => {
const page = await browser.newPage();
export const playwrightFetch = async (url: string, context) => {
const page = await context.newPage();
await page.setExtraHTTPHeaders(headers);
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
logger.http(`Requesting ${url}`);
@ -45,8 +45,8 @@ export const playwrightFetch = async (url: string, browser) => {
return response;
};
export const getItem = async (item, browser) => {
const response = await playwrightFetch(`${baseUrl}/api/video/${item.contentId}`, browser);
export const getItem = async (item, context) => {
const response = await playwrightFetch(`${baseUrl}/api/video/${item.contentId}`, context);
const videoInfo: Video = response.video;
videoInfo.gallery = hdGallery(videoInfo.gallery);

15
lib/routes/kuaishou/profile.ts
View File

@ -25,8 +25,8 @@ The profile page of the user, which contains the user's information, videos, and
async function handler(ctx) {
const { principalId } = ctx.req.param();
const browser = await playwright();
const page = await browser.newPage();
const context = await playwright();
const page = await context.newPage();
let retryCount = 0;
let resolve;
@ -34,13 +34,12 @@ async function handler(ctx) {
const promise = new Promise((res) => {
resolve = res;
});
await page.setRequestInterception(true);
page.on('request', (req) => {
const resourceType = req.resourceType();
await page.route('**/*', (route) => {
const resourceType = route.request().resourceType();
if (resourceType === 'image' || resourceType === 'media' || resourceType === 'font' || resourceType === 'stylesheet' || resourceType === 'ping') {
req.abort();
route.abort();
} else {
req.continue();
route.continue();
}
});
page.on('response', async (res) => {
@ -69,7 +68,7 @@ async function handler(ctx) {
await page.goto(`https://live.kuaishou.com/profile/${principalId}`);
const resData = (await promise.catch((error) => error)) as any[];
await browser.close();
await context.close();
const data: Data = {
title: userInfo?.name ?? `${principalId}的作品 - 快手`,
// description: JSON.stringify(resData),

13
lib/routes/linkedin/posts.ts
View File

@ -27,12 +27,11 @@ export const route: Route = {
const company_id = ctx.req.param('company_id');
// Puppeteer setup
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
const url = new URL(`${BASE_URL}/company/${company_id}`);
@ -49,7 +48,7 @@ export const route: Route = {
const companyName = parseCompanyName($);
const posts = parseCompanyPosts($);
await browser.close();
await context.close();
return {
title: `LinkedIn - ${companyName}'s Posts`,

12
lib/routes/missav/new.tsx
View File

@ -39,17 +39,17 @@ async function handler() {
const baseUrl = 'https://missav.ws';
const url = `${baseUrl}/${urlPath}`;
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',
});
const response = await page.evaluate(() => document.documentElement.innerHTML);
await browser.close();
await context.close();
// const response = await ofetch(`${baseUrl}/dm397/new`, {
// headers: {

6
lib/routes/nhentai/util.tsx
View File

@ -76,9 +76,9 @@ const fetchPage = async (url: string): Promise<string> => {
const { page, destroy } = await getPlaywrightPage(url, {
onBeforeLoad: async (page) => {
const allowedTypes = new Set(['document', 'script', 'xhr', 'fetch']);
await page.setRequestInterception(true);
page.on('request', (request) => {
allowedTypes.has(request.resourceType()) ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
allowedTypes.has(request.resourceType()) ? route.continue() : route.abort();
});
},
});

14
lib/routes/njust/utils.ts
View File

@ -3,9 +3,9 @@ import playwright from '@/utils/playwright';
async function getContent(url, pptr = false) {
if (pptr) {
const browser = await playwright();
const context = await playwright();
try {
const page = await browser.newPage();
const page = await context.newPage();
// 更改 window.navigator.webdriver 值以避开反爬
// await page.evaluateOnNewDocument(() => {
// // eslint-disable-next-line no-undef
@ -13,17 +13,17 @@ async function getContent(url, pptr = false) {
// get: () => undefined,
// });
// });
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'networkidle0',
waitUntil: 'networkidle',
});
const content = await page.content();
return content;
} finally {
await browser.close();
await context.close();
}
} else {
const response = await got(url);

14
lib/routes/nuaa/utils/pypasswaf.ts
View File

@ -6,18 +6,18 @@ import { getCookies } from '@/utils/playwright-utils';
* @desc 返回一个可用的cookie，使用 `got` 发起请求的时候，传入到`options.headers.cookie`即可
*/
export default async function getCookie(host) {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(host, {
waitUntil: 'networkidle0',
waitUntil: 'networkidle',
});
const cookie = await getCookies(page);
await browser.close();
await context.close();
return cookie;
}

6
lib/routes/nytimes/index.ts
View File

@ -81,7 +81,7 @@ async function handler(ctx) {
// Do nothing
}
const browser = await playwright();
const context = await playwright();
const feed = await parser.parseURL(rssUrl);
const items = await Promise.all(
feed.items.splice(0, 10).map(async (item) => {
@ -122,7 +122,7 @@ async function handler(ctx) {
hasEnVersion = true;
link = $('.dual-btn a').last().attr().href;
response = await utils.PuppeterGetter(ctx, browser, link);
response = await utils.PuppeterGetter(ctx, context, link);
}
}
}
@ -154,7 +154,7 @@ async function handler(ctx) {
})
);
await browser.close();
await context.close();
return {
title,

10
lib/routes/nytimes/utils.ts
View File

@ -15,12 +15,12 @@ const ProcessImage = ($, e) => {
return cover;
};
const PuppeterGetter = async (ctx, browser, link) => {
const PuppeterGetter = async (ctx, context, link) => {
const result = await cache.tryGet(`nyt: ${link}`, async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await page.goto(link, {
waitUntil: 'domcontentloaded',

12
lib/routes/oceanengine/arithmetic-index.tsx
View File

@ -111,15 +111,15 @@ async function handler(ctx) {
const item = await cache.tryGet(
link,
async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? route.continue() : route.abort();
});
await page.goto('https://trendinsight.oceanengine.com/arithmetic-index');
const res = await getMultiKeywordHotTrend(page, keyword, start_date, end_date, channel);
await browser.close();
await context.close();
const rawData = JSON.parse(res).data;
const data = decrypt(rawData).hot_list[0];

12
lib/routes/parliament.uk/commonslibrary.ts
View File

@ -26,11 +26,11 @@ async function handler(ctx) {
const { topic } = ctx.req.param();
const baseUrl = 'https://commonslibrary.parliament.uk';
const url = `${baseUrl}/type/${topic}/`;
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',
@ -47,7 +47,7 @@ async function handler(ctx) {
description: $(article).find('p').last().text().trim(),
pubDate: timezone($(article).find('.card__date time').attr('datetime')),
}));
await browser.close();
await context.close();
return {
title: `parliament - lordslibrary - ${topic}`,
link: url,

12
lib/routes/parliament.uk/lordslibrary.ts
View File

@ -26,11 +26,11 @@ async function handler(ctx) {
const { topic } = ctx.req.param();
const baseUrl = 'https://lordslibrary.parliament.uk';
const url = `${baseUrl}/type/${topic}/`;
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',
@ -47,7 +47,7 @@ async function handler(ctx) {
description: $(article).find('p').last().text().trim(),
pubDate: timezone($(article).find('.card__date time').attr('datetime')),
}));
await browser.close();
await context.close();
return {
title: `parliament - lordslibrary - ${topic}`,
link: url,

16
lib/routes/perplexity/blog.ts
View File

@ -38,11 +38,11 @@ async function handler(ctx: Context) {
const limit = Number.parseInt(ctx.req.query('limit') ?? '20', 10);
const rootUrl = 'https://www.perplexity.ai/hub';
const { page, destroy, browser } = await getPlaywrightPage(rootUrl, {
const { page, destroy, context } = await getPlaywrightPage(rootUrl, {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
},
});
@ -112,11 +112,11 @@ async function handler(ctx: Context) {
}
return (await cache.tryGet(item.link, async () => {
const contentPage = await browser.newPage();
const contentPage = await context.newPage();
await contentPage.setRequestInterception(true);
contentPage.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await contentPage.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await contentPage.goto(item.link!, {

16
lib/routes/perplexity/changelog.ts
View File

@ -16,11 +16,11 @@ export const handler = async (ctx: Context): Promise<Data> => {
logger.http(`Fetching Perplexity changelog from ${targetUrl}`);
const { page, destroy, browser } = await getPlaywrightPage(targetUrl, {
const { page, destroy, context } = await getPlaywrightPage(targetUrl, {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
},
});
@ -101,12 +101,12 @@ export const handler = async (ctx: Context): Promise<Data> => {
logger.http(`Fetching full content for ${item.link!}`);
// Create a new page in the same browser session
const contentPage = await browser.newPage();
const contentPage = await context.newPage();
// Set request interception for this page
await contentPage.setRequestInterception(true);
contentPage.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await contentPage.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
// Navigate to the item link

6
lib/routes/picnob/utils.ts
View File

@ -4,9 +4,9 @@ const playwrightGet = async (url) => {
let data;
const { destroy } = await getPlaywrightPage(url, {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
page.on('response', async (response) => {
data = await (response.request().url().includes('/api/posts') ? response.json() : response.text());

6
lib/routes/picuki/profile.ts
View File

@ -88,9 +88,9 @@ async function handler(ctx) {
const { page, destroy } = await getPlaywrightPage(profileUrl, {
onBeforeLoad: async (page) => {
const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']);
await page.setRequestInterception(true);
page.on('request', (request) => {
expectResourceTypes.has(request.resourceType()) ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
expectResourceTypes.has(request.resourceType()) ? route.continue() : route.abort();
});
},
});

12
lib/routes/pincong/utils.ts
View File

@ -4,17 +4,17 @@ const baseUrl = 'https://pincong.rocks';
const playwrightGet = (url, cache) =>
cache.tryGet(url, async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',
});
const html = await page.evaluate(() => document.documentElement.innerHTML);
await browser.close();
await context.close();
return html;
});

12
lib/routes/pnas/index.tsx
View File

@ -53,16 +53,16 @@ async function handler(ctx) {
};
});
const browser = await playwright();
const context = await playwright();
const out = await Promise.all(
list.map((item) =>
cache.tryGet(item.link, async () => {
const page = await browser.newPage();
const page = await context.newPage();
await setCookies(page, await cookieJar.getCookieString(item.link), '.pnas.org');
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
logger.http(`Requesting ${item.link}`);
await page.goto(item.link, {
@ -112,7 +112,7 @@ async function handler(ctx) {
)
);
await browser.close();
await context.close();
return {
title: `${$('.banner-widget__content h1').text()} - PNAS`,

20
lib/routes/researchgate/publications.ts
View File

@ -23,11 +23,11 @@ async function handler(ctx) {
const rootUrl = 'https://www.researchgate.net';
const currentUrl = `${rootUrl}/profile/${id}`;
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(currentUrl);
const response = await page.evaluate(() => document.documentElement.innerHTML);
@ -49,10 +49,10 @@ async function handler(ctx) {
const items = await Promise.all(
list.map((item) =>
cache.tryGet(item.link, async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(item.link);
const detailResponse = await page.evaluate(() => document.documentElement.innerHTML);
@ -77,7 +77,7 @@ async function handler(ctx) {
)
);
await browser.close();
await context.close();
return {
title: `${$('meta[property="profile:username"]').attr('content')}'s Publications - ResearchGate`,

13
lib/routes/science/blogs.ts
View File

@ -40,12 +40,11 @@ async function handler(ctx) {
const response = await cache.tryGet(
link,
async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await page.goto(link, {
@ -55,7 +54,7 @@ async function handler(ctx) {
const response = await page.content();
await page.close();
await browser.close();
await context.close();
return response;
},
config.cache.routeExpire,

6
lib/routes/science/current.ts
View File

@ -66,9 +66,9 @@ async function handler(ctx) {
.toArray()
.map((item) => getItem(item, $));
const browser = await playwright();
const items = await fetchDesc(list, browser, cache.tryGet);
await browser.close();
const context = await playwright();
const items = await fetchDesc(list, context, cache.tryGet);
await context.close();
return {
title: `${pageTitleName} | Current Issue`,

6
lib/routes/science/early.ts
View File

@ -47,9 +47,9 @@ async function handler(ctx) {
.toArray()
.map((item) => getItem(item, $));
const browser = await playwright();
const items = await fetchDesc(list, browser, cache.tryGet);
await browser.close();
const context = await playwright();
const items = await fetchDesc(list, context, cache.tryGet);
await context.close();
return {
title: $('head title').text(),

10
lib/routes/science/utils.tsx
View File

@ -6,14 +6,14 @@ import { parseDate } from '@/utils/parse-date';
const baseUrl = 'https://www.science.org';
const fetchDesc = (list, browser, tryGet) =>
const fetchDesc = (list, context, tryGet) =>
Promise.all(
list.map((item) =>
tryGet(item.link, async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(item.link, {

12
lib/routes/sotwe/user.ts
View File

@ -61,11 +61,11 @@ async function handler(ctx) {
const data = await cache.tryGet(
`sotwe:user:${id}`,
async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
['document', 'script', 'xhr', 'fetch'].includes(request.resourceType()) ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
['document', 'script', 'xhr', 'fetch'].includes(request.resourceType()) ? route.continue() : route.abort();
});
const apiUrl = `${baseUrl}/api/v3/user/${id}/`;
logger.http(`Requesting ${apiUrl}`);
@ -74,7 +74,7 @@ async function handler(ctx) {
});
const response = await page.evaluate(() => document.documentElement.textContent);
await page.close();
await browser.close();
await context.close();
return JSON.parse(response || '{}');
},

12
lib/routes/spankbang/new-videos.tsx
View File

@ -22,15 +22,15 @@ const handler = async () => {
const baseUrl = 'https://spankbang.com';
const link = `${baseUrl}/new_videos/`;
const browser = await playwright();
const context = await playwright();
const data = await cache.tryGet(
link,
async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
logger.http(`Requesting ${link}`);
await page.goto(link, {
@ -67,7 +67,7 @@ const handler = async () => {
false
);
await browser.close();
await context.close();
return {
title: data.title,

14
lib/routes/tiktok/user.ts
View File

@ -43,12 +43,12 @@ async function handler(ctx) {
const data = await cache.tryGet(
`tiktok:user:${user}`,
async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
const context = await playwright();
const page = await context.newPage();
let itemList = { itemList: [] };
page.on('request', (request) => {
['document', 'script', 'xhr', 'fetch'].includes(request.resourceType()) ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
['document', 'script', 'xhr', 'fetch'].includes(request.resourceType()) ? route.continue() : route.abort();
});
page.on('response', async (response) => {
const request = response.request();
@ -57,11 +57,11 @@ async function handler(ctx) {
}
});
await page.goto(`${baseUrl}/${user}`, {
waitUntil: 'networkidle0',
waitUntil: 'networkidle',
});
const pageHtml = await page.content();
await browser.close();
await context.close();
const $ = load(pageHtml);
const rehydrationData = JSON.parse($('script#__UNIVERSAL_DATA_FOR_REHYDRATION__').text());

4
lib/routes/toutiao/a-bogus.ts
View File

@ -1,6 +1,6 @@
// oxlint-disable no-undef
/* eslint-disable unicorn/prefer-spread */
/* eslint-disable unicorn/prefer-math-trunc */
// oxlint-disable unicorn/prefer-spread
// oxlint-disable unicorn/prefer-math-trunc
// @ts-nocheck
// Credits:

8
lib/routes/twitter/api/web-api/login.ts
View File

@ -29,8 +29,8 @@ async function login({ username, password, authenticationSecret }) {
await loginLimiterQueue.removeTokens(1);
const cookieJar = new CookieJar();
const browser = await playwright();
const page = await browser.newPage();
const context = await playwright();
const page = await context.newPage();
await page.goto('https://x.com/i/flow/login');
await page.waitForSelector('input[autocomplete="username"]');
await page.type('input[autocomplete="username"]', username);
@ -54,7 +54,7 @@ async function login({ username, password, authenticationSecret }) {
logger.error(`twitter debug: twitter username ${username} login failed: messageprompt-suspended-prompt`);
resolve('');
}
const cookies = await page.cookies();
const cookies = await page.context().cookies();
for (const cookie of cookies) {
cookieJar.setCookieSync(`${cookie.name}=${cookie.value}`, 'https://x.com');
}
@ -64,7 +64,7 @@ async function login({ username, password, authenticationSecret }) {
});
});
const cookieString = await waitForRequest;
await browser.close();
await context.close();
return cookieString;
} catch (error) {
logger.error(`twitter debug: twitter username ${username} login failed:`, error);

20
lib/routes/uchicago/current.ts
View File

@ -35,11 +35,11 @@ async function handler(ctx) {
const baseUrl = 'https://www.journals.uchicago.edu';
const link = `${baseUrl}/toc/${journal}/current`;
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
logger.http(`Requesting ${link}`);
await page.goto(link, {
@ -57,11 +57,11 @@ async function handler(ctx) {
const items = await Promise.all(
list.map((item) =>
cache.tryGet(item.link, async () => {
const page = await browser.newPage();
const page = await context.newPage();
await setCookies(page, cookies, 'journals.uchicago.edu');
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
logger.http(`Requesting ${item.link}`);
await page.goto(item.link, {
@ -95,7 +95,7 @@ async function handler(ctx) {
)
);
await browser.close();
await context.close();
return {
title: $('head title').text(),

14
lib/routes/uestc/auto.ts
View File

@ -32,17 +32,17 @@ export const route: Route = {
};
async function handler() {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(baseIndexUrl, {
waitUntil: 'networkidle2',
waitUntil: 'networkidle',
});
const content = await page.content();
await browser.close();
await context.close();
const $ = load(content);

14
lib/routes/uestc/cqe.ts
View File

@ -52,17 +52,17 @@ async function handler(ctx) {
throw new InvalidParameterError('type not supported');
}
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(baseUrl + pageUrl, {
waitUntil: 'networkidle2',
waitUntil: 'networkidle',
});
const content = await page.content();
await browser.close();
await context.close();
const $ = load(content);

14
lib/routes/uestc/scse.ts
View File

@ -46,17 +46,17 @@ export const route: Route = {
};
async function handler() {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(baseIndexUrl, {
waitUntil: 'networkidle2',
waitUntil: 'networkidle',
});
const content = await page.content();
await browser.close();
await context.close();
const $ = load(content);

14
lib/routes/uestc/sice.ts
View File

@ -33,17 +33,17 @@ export const route: Route = {
};
async function handler() {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(baseIndexUrl, {
waitUntil: 'networkidle2',
waitUntil: 'networkidle',
});
const content = await page.content();
await browser.close();
await context.close();
const $ = load(content);

14
lib/routes/uestc/sise.ts
View File

@ -67,17 +67,17 @@ async function handler(ctx) {
throw new InvalidParameterError('type not supported');
}
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' ? route.continue() : route.abort();
});
await page.goto(baseUrl, {
waitUntil: 'networkidle2',
waitUntil: 'networkidle',
});
const content = await page.content();
await browser.close();
await context.close();
const $ = load(content);

16
lib/routes/ups/track.ts
View File

@ -26,17 +26,15 @@ async function handler(ctx) {
const { trackingNumber } = ctx.req.param();
const url = `https://www.ups.com/track?loc=en_US&tracknum=${trackingNumber}`;
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
const context = await playwright();
const page = await context.newPage();
// skip loading images, stylesheets, and fonts
page.on('request', (request) => {
if (['image', 'stylesheet', 'font', 'ping', 'fetch'].includes(request.resourceType())) {
request.abort();
await page.route('**/*', (route) => {
if (['image', 'stylesheet', 'font', 'ping', 'fetch'].includes(route.request().resourceType())) {
route.abort();
} else {
request.continue();
route.continue();
}
});
@ -57,7 +55,7 @@ async function handler(ctx) {
await page.waitForSelector('tr[id^="stApp_activitydetails_row"]');
const content = await page.content();
await browser.close();
await context.close();
const $ = load(content);

18
lib/routes/uraaka-joshi/uraaka-joshi-user.ts
View File

@ -38,14 +38,18 @@ async function handler(ctx) {
const response = await cache.tryGet(
link,
async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' ? route.continue() : route.abort();
});
page.on('requestfinished', async (request) => {
if (request.url() === link && request.response().status() === 403) {
if (request.url() !== link) {
return;
}
const response = await request.response();
if (response?.status() === 403) {
await page.close();
}
});
@ -63,7 +67,7 @@ async function handler(ctx) {
} catch {
throw new Error('Access denied (403)');
}
await browser.close();
await context.close();
return html;
},
config.cache.routeExpire,

18
lib/routes/uraaka-joshi/uraaka-joshi.ts
View File

@ -25,15 +25,19 @@ async function handler() {
const link = 'https://www.uraaka-joshi.com/';
const title = '裏垢女子まとめ';
const browser = await playwright();
const context = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' ? route.continue() : route.abort();
});
page.on('requestfinished', async (request) => {
if (request.url() === link && request.response().status() === 403) {
if (request.url() !== link) {
return;
}
const response = await request.response();
if (response?.status() === 403) {
await page.close();
}
});
@ -53,7 +57,7 @@ async function handler() {
} catch {
throw new Error('Access denied (403)');
}
await browser.close();
await context.close();
const $ = load(html);
const list = $('.grid-cell');

13
lib/routes/weibo/utils.ts
View File

@ -83,22 +83,21 @@ const weiboUtils = {
const { page, destroy } = await getPlaywrightPage(url, {
onBeforeLoad: async (page) => {
const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']);
await page.setUserAgent(weiboUtils.apiHeaders['User-Agent']);
await page.setRequestInterception(true);
page.on('request', (request) => {
await page.setExtraHTTPHeaders({ 'User-Agent': weiboUtils.apiHeaders['User-Agent'] });
await page.route('**/*', (route) => {
const request = route.request();
// 1st: initial request, 302 to visitor.passport.weibo.cn; 2nd: auth ok
if (!expectResourceTypes.has(request.resourceType()) || times >= 2) {
request.abort();
route.abort();
return;
}
if (request.url().startsWith(url)) {
times++;
}
request.continue();
route.continue();
});
},
// networkidle2 returns too early if the connection is slow
gotoConfig: { waitUntil: 'networkidle0' },
gotoConfig: { waitUntil: 'networkidle' },
});
const cookies: string = await getCookies(page, 'weibo.cn');
await destroy();

18
lib/routes/xiaohongshu/util.ts
View File

@ -65,9 +65,9 @@ const getUser = (url, cache) =>
// Use Playwright
const { page, destroy } = await getPlaywrightPage(url, {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' || request.resourceType() === 'other' ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' || request.resourceType() === 'other' ? route.continue() : route.abort();
});
},
});
@ -128,12 +128,12 @@ const getBoard = (url, cache) =>
}
// Use Playwright
const browser = await playwright();
const context = await playwright();
try {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? route.continue() : route.abort();
});
logger.http(`Requesting ${url}`);
await page.goto(url);
@ -141,7 +141,7 @@ const getBoard = (url, cache) =>
const initialSsrState = await page.evaluate(() => (window as any).__INITIAL_SSR_STATE__);
return initialSsrState.Main;
} finally {
await browser.close();
await context.close();
}
},
config.cache.routeExpire,

6
lib/routes/xsijishe/rank.ts
View File

@ -60,10 +60,10 @@ async function handler(ctx) {
throw new InvalidParameterError('Invalid rank type');
}
const browser = await playwright();
const context = await playwright();
const url = `${baseUrl}/portal.php`;
try {
const data = await playwrightGet(url, browser, '.nex_recon_lists', {
const data = await playwrightGet(url, context, '.nex_recon_lists', {
cookie: config.xsijishe.cookie,
userAgent: config.xsijishe.userAgent,
});
@ -97,6 +97,6 @@ async function handler(ctx) {
item: items,
};
} finally {
await browser.close();
await context.close();
}
}

15
lib/routes/xsijishe/utils.ts
View File

@ -1,4 +1,5 @@
import type { Browser } from '@/utils/playwright';
import type { BrowserContext } from 'patchright';
import { setCookies } from '@/utils/playwright-utils';
interface PlaywrightGetOptions {
@ -7,21 +8,21 @@ interface PlaywrightGetOptions {
userAgent?: string;
}
const playwrightGet = async (url: string, browser: Browser, waitForSelector = '.t_f', options: PlaywrightGetOptions = {}) => {
const page = await browser.newPage();
const playwrightGet = async (url: string, context: BrowserContext, waitForSelector = '.t_f', options: PlaywrightGetOptions = {}) => {
const page = await context.newPage();
const expectResourceTypes = new Set(['document', 'script']);
try {
if (options.userAgent) {
await page.setUserAgent(options.userAgent);
await page.setExtraHTTPHeaders({ 'User-Agent': options.userAgent });
}
if (options.cookie) {
await setCookies(page, options.cookie, 'xsijishe.com');
}
await page.setRequestInterception(true);
page.on('request', (request) => {
expectResourceTypes.has(request.resourceType()) ? request.continue() : request.abort();
await page.route('**/*', (route) => {
const request = route.request();
expectResourceTypes.has(request.resourceType()) ? route.continue() : route.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',

10
lib/routes/xueqiu/cookies.ts
View File

@ -7,11 +7,11 @@ export const parseToken = (link: string) =>
cache.tryGet(
'xueqiu:token',
async () => {
const browser = await playwright();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
const context = await playwright();
const page = await context.newPage();
await page.route('**/*', (route) => {
const request = route.request();
request.resourceType() === 'document' ? route.continue() : route.abort();
});
await page.goto(link, {
waitUntil: 'domcontentloaded',

6
lib/routes/xueqiu/user.ts
View File

@ -50,9 +50,9 @@ async function handler(ctx) {
const link = `${rootUrl}/u/${id}`;
const token = await parseToken(link);
const browser = await playwright();
const context = await playwright();
try {
const mainPage = await browser.newPage();
const mainPage = await context.newPage();
await mainPage.setExtraHTTPHeaders({
Cookie: token as string,
@ -161,6 +161,6 @@ async function handler(ctx) {
item: items,
};
} finally {
await browser.close();
await context.close();
}
}

4
lib/routes/zhihu/execlib/x-zse-96-v3.ts
View File

@ -1,4 +1,4 @@
/* eslint-disable unicorn/prefer-code-point */
// oxlint-disable unicorn/prefer-code-point
// Credit:
// https://blog.csdn.net/zjq592767809/article/details/126512798
// https://blog.csdn.net/zhoumi_/article/details/126659351
@ -29,7 +29,6 @@ const __g = {
for (let o = e.slice(16 * i, 16 * (i + 1)), c = 0; c < 16; c++) {
a[c] = o[c] ^ t[c];
}
// eslint-disable-next-line unicorn/prefer-spread
((t = __g.r(a)), (n = n.concat(t)), i++);
}
return n;
@ -95,7 +94,6 @@ const preProcess = (md5Str) => {
const __g_r = __g.r(new_md5_charCodeAt_arr);
const md5CharCodeAtBackArr = md5CharCodeAtArr.slice(16, 48);
const __g_x = __g.x(md5CharCodeAtBackArr, __g_r);
// eslint-disable-next-line unicorn/prefer-spread
return __g_r.concat(__g_x);
};

40
lib/utils/playwright-utils.test.ts
View File

@ -1,15 +1,15 @@
import type { BrowserContext } from 'patchright';
import { afterEach, describe, expect, it, vi } from 'vitest';
import type { Browser } from '@/utils/playwright';
import playwright from '@/utils/playwright';
import { constructCookieArray, getCookies, parseCookieArray, setCookies } from '@/utils/playwright-utils';
let browser: Browser | null = null;
let context: BrowserContext | null = null;
afterEach(async () => {
if (browser) {
await browser.close();
browser = null;
if (context) {
await context.close();
context = null;
}
vi.resetModules();
@ -17,19 +17,19 @@ afterEach(async () => {
describe('browser cookie utils', () => {
const cookieArrayExampleCom = [
{ name: 'foobar', value: '', domain: 'example.com' },
{ name: 'foo', value: 'bar', domain: 'example.com' },
{ name: 'baz', value: 'qux', domain: 'example.com' },
{ name: 'foobar', value: '', domain: 'example.com', path: '/' },
{ name: 'foo', value: 'bar', domain: 'example.com', path: '/' },
{ name: 'baz', value: 'qux', domain: 'example.com', path: '/' },
];
const cookieArraySubExampleCom = [
{ name: 'barfoo', value: '', domain: 'sub.example.com' },
{ name: 'bar', value: 'foo', domain: 'sub.example.com' },
{ name: 'qux', value: 'baz', domain: 'sub.example.com' },
{ name: 'barfoo', value: '', domain: 'sub.example.com', path: '/' },
{ name: 'bar', value: 'foo', domain: 'sub.example.com', path: '/' },
{ name: 'qux', value: 'baz', domain: 'sub.example.com', path: '/' },
];
const cookieArrayRsshubTest = [
{ name: '', value: 'rsshub', domain: 'rsshub.test' },
{ name: 'rsshub', value: '', domain: 'rsshub.test' },
{ name: 'test', value: 'rsshub', domain: 'rsshub.test' },
{ name: '', value: 'rsshub', domain: 'rsshub.test', path: '/' },
{ name: 'rsshub', value: '', domain: 'rsshub.test', path: '/' },
{ name: 'test', value: 'rsshub', domain: 'rsshub.test', path: '/' },
];
const cookieArrayAll = [...cookieArrayExampleCom, ...cookieArraySubExampleCom, ...cookieArrayRsshubTest];
@ -69,8 +69,8 @@ describe('browser cookie utils', () => {
});
it('getCookies httpbingo', async () => {
browser = await playwright();
const page = await browser.newPage();
context = await playwright();
const page = await context.newPage();
await page.goto('https://httpbingo.org/cookies/set?foo=bar&baz=qux', {
waitUntil: 'domcontentloaded',
});
@ -78,8 +78,8 @@ describe('browser cookie utils', () => {
}, 45000);
it('setCookies httpbingo', async () => {
browser = await playwright();
const page = await browser.newPage();
context = await playwright();
const page = await context.newPage();
// httpbingo.org cannot recognize cookies with empty name properly, so we cannot use cookieStrAll here
await setCookies(page, cookieStrExampleCom, 'httpbingo.org');
await page.goto('https://httpbingo.org/cookies', {
@ -90,8 +90,8 @@ describe('browser cookie utils', () => {
}, 45000);
it('setCookies & getCookies example.org', async () => {
browser = await playwright();
const page = await browser.newPage();
context = await playwright();
const page = await context.newPage();
// we can use cookieStrAll here!
await setCookies(page, cookieStrAll, 'example.org');
await page.goto('https://example.org', {

6
lib/utils/playwright-utils.ts
View File

@ -27,7 +27,7 @@ const parseCookieArray = (cookies, domainFilter?: string | RegExp) => {
const constructCookieArray = (cookieStr, domain) =>
cookieStr.split('; ').map((item) => {
const [name, value] = item.split('=');
return value === undefined ? { name: '', value: name, domain } : { name, value, domain };
return value === undefined ? { name: '', value: name, domain, path: '/' } : { name, value, domain, path: '/' };
});
/**
@ -40,7 +40,7 @@ const constructCookieArray = (cookieStr, domain) =>
*/
const setCookies = async (page, cookieStr, domain) => {
const cookies = constructCookieArray(cookieStr, domain);
await page.setCookie(...cookies);
await page.context().addCookies(cookies);
};
/**
@ -51,7 +51,7 @@ const setCookies = async (page, cookieStr, domain) => {
* @return {Promise<string>} Cookie-header-style cookie string
*/
const getCookies = async (page, domainFilter?: string) => {
const cookies = await page.cookies();
const cookies = await page.context().cookies();
return parseCookieArray(cookies, domainFilter);
};

133
lib/utils/playwright.mock.test.ts
View File

@ -4,19 +4,6 @@ const connect = vi.fn();
const connectOverCDP = vi.fn();
const launch = vi.fn();
const routeContinue = vi.fn();
const routeAbort = vi.fn();
const route = {
request: vi.fn(),
continue: routeContinue,
abort: routeAbort,
};
const request = {
resourceType: vi.fn(),
url: vi.fn(),
};
let page: any;
let context: any;
let browser: any;
@ -26,9 +13,7 @@ const createBrowserMocks = () => {
context: vi.fn(),
goto: vi.fn(),
on: vi.fn(),
route: vi.fn(),
setExtraHTTPHeaders: vi.fn(),
unroute: vi.fn(),
};
context = {
@ -53,7 +38,7 @@ const proxyMock = {
getDispatcherForProxy: vi.fn(),
};
vi.mock('playwright', () => ({
vi.mock('patchright', () => ({
chromium: {
connect,
connectOverCDP,
@ -83,11 +68,6 @@ const resetMocks = () => {
connect.mockReset();
connectOverCDP.mockReset();
launch.mockReset();
routeContinue.mockReset();
routeAbort.mockReset();
route.request.mockReset();
request.resourceType.mockReset();
request.url.mockReset();
proxyMock.multiProxy = undefined;
proxyMock.getCurrentProxy.mockReset();
proxyMock.markProxyFailed.mockReset();
@ -110,7 +90,7 @@ describe('getPlaywrightPage (mocked)', () => {
const getPlaywrightPage = await loadPlaywright();
const onBeforeLoad = vi.fn();
const close = browser.close;
const contextClose = context.close;
const result = await getPlaywrightPage('https://example.com', {
noGoto: true,
onBeforeLoad,
@ -118,17 +98,39 @@ describe('getPlaywrightPage (mocked)', () => {
const endpoint = connect.mock.calls[0][0] as string;
expect(connectOverCDP).not.toHaveBeenCalled();
expect(endpoint).toContain('launch-options=');
expect(endpoint).not.toContain('launch=');
const launchOptions = JSON.parse(new URL(endpoint).searchParams.get('launch-options') || '{}');
expect(endpoint).toContain('launch=');
expect(endpoint).not.toContain('launch-options=');
const launchOptions = JSON.parse(new URL(endpoint).searchParams.get('launch') || '{}');
expect(launchOptions.args).not.toContainEqual(expect.stringContaining('--user-agent='));
expect(launchOptions.executablePath).toBeUndefined();
expect(launchOptions.acceptInsecureCerts).toBe(true);
expect(onBeforeLoad).toHaveBeenCalled();
await result.destroy();
expect(close).toHaveBeenCalled();
expect(contextClose).toHaveBeenCalled();
});
it('does not override the browser user agent', async () => {
it('merges browserless launch options with existing ws endpoint launch param', async () => {
resetMocks();
connect.mockResolvedValue(browser);
launch.mockResolvedValue(browser);
page.goto.mockResolvedValue(undefined);
browser.close.mockResolvedValue(undefined);
process.env.PLAYWRIGHT_WS_ENDPOINT = `ws://localhost:3000/?token=abc&launch=${encodeURIComponent(JSON.stringify({ stealth: true }))}`;
proxyMock.getCurrentProxy.mockReturnValue(null);
const getPlaywrightPage = await loadPlaywright();
const result = await getPlaywrightPage('https://example.com', { noGoto: true });
const endpoint = connect.mock.calls[0][0] as string;
const launchOptions = JSON.parse(new URL(endpoint).searchParams.get('launch') || '{}');
expect(launchOptions.stealth).toBe(true);
expect(launchOptions.headless).toBe(true);
await result.destroy();
});
it('does override the default HeadlessChrome user agent', async () => {
resetMocks();
launch.mockResolvedValue(browser);
page.goto.mockResolvedValue(undefined);
@ -143,10 +145,12 @@ describe('getPlaywrightPage (mocked)', () => {
})
);
expect(browser.newContext).toHaveBeenCalledWith(
expect.not.objectContaining({
expect.objectContaining({
userAgent: expect.any(String),
})
);
const contextOptions = browser.newContext.mock.calls[0][0];
expect(contextOptions.userAgent).not.toMatch(/HeadlessChrome/i);
});
it('supports extending the browser auto close timeout', async () => {
@ -175,33 +179,6 @@ describe('getPlaywrightPage (mocked)', () => {
}
});
it('handles requestfinished events after the remote page closes', async () => {
resetMocks();
launch.mockResolvedValue(browser);
page.goto.mockResolvedValue(undefined);
proxyMock.getCurrentProxy.mockReturnValue(null);
const getPlaywrightPage = await loadPlaywright();
const handler = vi.fn();
const rawOn = page.on;
await getPlaywrightPage('https://example.com', {
onBeforeLoad: (page) => {
page.on('requestfinished', handler);
},
});
const finishedHandler = rawOn.mock.calls.find(([event]) => event === 'requestfinished')?.[1];
await finishedHandler?.({
response: vi.fn().mockRejectedValue(new Error('closed')),
url: () => 'https://example.com/api',
});
expect(handler).toHaveBeenCalledTimes(1);
const finishedRequest = handler.mock.calls[0][0];
expect(finishedRequest.response()).toBeNull();
expect(finishedRequest.url()).toBe('https://example.com/api');
});
it('marks proxy failed when navigation throws with multi-proxy', async () => {
resetMocks();
launch.mockResolvedValue(browser);
@ -219,50 +196,4 @@ describe('getPlaywrightPage (mocked)', () => {
expect(proxyMock.markProxyFailed).toHaveBeenCalledWith(currentProxy.uri);
});
it('maps legacy networkidle waits to playwright networkidle', async () => {
resetMocks();
launch.mockResolvedValue(browser);
page.goto.mockResolvedValue(undefined);
proxyMock.getCurrentProxy.mockReturnValue(null);
const getPlaywrightPage = await loadPlaywright();
const goto = page.goto;
await getPlaywrightPage('https://example.com', {
gotoConfig: {
waitUntil: 'networkidle2',
},
});
expect(goto).toHaveBeenCalledWith('https://example.com', {
waitUntil: 'networkidle',
});
});
it('keeps legacy request interception helpers', async () => {
resetMocks();
launch.mockResolvedValue(browser);
page.goto.mockResolvedValue(undefined);
proxyMock.getCurrentProxy.mockReturnValue(null);
request.resourceType.mockReturnValue('image');
request.url.mockReturnValue('https://example.com/logo.png');
routeAbort.mockReturnValueOnce(new Promise((resolve) => setTimeout(resolve, 0)));
route.request.mockReturnValue(request);
const getPlaywrightPage = await loadPlaywright();
await getPlaywrightPage('https://example.com', {
onBeforeLoad: async (page) => {
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
});
},
});
const routeHandler = page.route.mock.calls[0][1];
await routeHandler(route);
expect(routeAbort).toHaveBeenCalled();
expect(routeContinue).not.toHaveBeenCalled();
});
});

31
lib/utils/playwright.test.ts
View File

@ -1,15 +1,14 @@
import type { BrowserContext } from 'patchright';
import { afterEach, describe, expect, it, vi } from 'vitest';
import type { Browser } from '@/utils/playwright';
import wait from './wait';
let browser: Browser | null = null;
let context: BrowserContext | null = null;
afterEach(async () => {
if (browser) {
await browser.close();
browser = null;
if (context) {
await context.close();
context = null;
}
delete process.env.PROXY_URI;
@ -25,9 +24,10 @@ afterEach(async () => {
describe('playwright', () => {
it('playwright run', async () => {
const { default: playwright } = await import('./playwright');
browser = await playwright();
context = await playwright();
const browser = context.browser();
const startTime = Date.now();
const page = await browser.newPage();
const page = await context.newPage();
await page.goto('https://www.google.com', {
waitUntil: 'domcontentloaded',
});
@ -35,13 +35,13 @@ describe('playwright', () => {
const html = await page.evaluate(() => document.body.innerHTML);
expect(html.length).toBeGreaterThan(0);
expect(browser.isConnected()).toBe(true);
expect(browser?.isConnected()).toBe(true);
const sleepTime = 31 * 1000 - (Date.now() - startTime);
if (sleepTime > 0) {
await wait(sleepTime);
}
expect(browser.isConnected()).toBe(false);
browser = null;
expect(browser?.isConnected()).toBe(false);
context = null;
}, 45000);
});
@ -50,18 +50,19 @@ describe('getPlaywrightPage', () => {
const { getPlaywrightPage } = await import('./playwright');
const playwright = await getPlaywrightPage('https://www.google.com');
const page = playwright.page;
browser = playwright.browser;
context = playwright.context;
const browser = context.browser();
const startTime = Date.now();
const html = await page.evaluate(() => document.body.innerHTML);
expect(html.length).toBeGreaterThan(0);
expect(browser.isConnected()).toBe(true);
expect(browser?.isConnected()).toBe(true);
const sleepTime = 31 * 1000 - (Date.now() - startTime);
if (sleepTime > 0) {
await wait(sleepTime);
}
expect(browser.isConnected()).toBe(false);
browser = null;
expect(browser?.isConnected()).toBe(false);
context = null;
}, 45000);
});

262
lib/utils/playwright.ts
View File

@ -1,66 +1,15 @@
import type { Browser as PlaywrightBrowser, BrowserContext, BrowserContextOptions, LaunchOptions, Page as PlaywrightPage, Request as PlaywrightRequest, Response as PlaywrightResponse, Route } from 'playwright';
import { chromium } from 'playwright';
import type { Browser, BrowserContext, BrowserContextOptions, LaunchOptions, Page } from 'patchright';
import { chromium } from 'patchright';
import { config } from '@/config';
import logger from './logger';
import proxy from './proxy';
type SetCookieParam = Parameters<BrowserContext['addCookies']>[0][number];
type Cookie = Awaited<ReturnType<BrowserContext['cookies']>>[number];
type GotoOptions = Parameters<PlaywrightPage['goto']>[1] & {
waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'networkidle0' | 'networkidle2';
};
type GotoOptions = Parameters<Page['goto']>[1];
type ProxyState = NonNullable<ReturnType<typeof proxy.getCurrentProxy>>;
type RouteRequest = {
abort: (errorCode?: string) => Promise<void>;
continue: (options?: Parameters<Route['continue']>[0]) => Promise<void>;
resourceType: () => ReturnType<PlaywrightRequest['resourceType']>;
url: () => string;
};
type FinishedRequest = {
response: () => {
status: () => number;
} | null;
url: () => string;
};
type RequestHandler = (request: RouteRequest) => Promise<void> | void;
type RequestFinishedHandler = (request: FinishedRequest) => Promise<void> | void;
type HandledRouteRequest = RouteRequest & { handled: boolean };
export type Page = PlaywrightPage & {
authenticate: (credentials: { password?: string; username?: string }) => Promise<void>;
cookies: (urls?: string | string[]) => Promise<Cookie[]>;
goto: (url: string, options?: GotoOptions) => ReturnType<PlaywrightPage['goto']>;
on: ((event: 'request', handler: RequestHandler) => Page) & ((event: 'requestfinished', handler: RequestFinishedHandler) => Page) & PlaywrightPage['on'];
setCookie: (...cookies: SetCookieParam[]) => Promise<void>;
setRequestInterception: (enabled: boolean) => Promise<void>;
setUserAgent: (userAgent: string) => Promise<void>;
};
export type Browser = PlaywrightBrowser & {
cookies: (urls?: string | string[]) => Promise<Cookie[]>;
newPage: () => Promise<Page>;
setCookie: (...cookies: SetCookieParam[]) => Promise<void>;
userAgent: () => string;
};
const normalizeWaitUntil = (waitUntil: GotoOptions['waitUntil']) => (waitUntil === 'networkidle0' || waitUntil === 'networkidle2' ? 'networkidle' : waitUntil);
const normalizeGotoOptions = (options?: GotoOptions): Parameters<PlaywrightPage['goto']>[1] | undefined =>
options
? {
...options,
waitUntil: normalizeWaitUntil(options.waitUntil),
}
: options;
const withDefaultCookiePath = (cookie: SetCookieParam): SetCookieParam => ('domain' in cookie && !('path' in cookie) ? { ...cookie, path: '/' } : cookie);
const proxyServerFromUrl = (proxyUrl: URL) => {
const protocol = proxyUrl.protocol.replace('socks5h:', 'socks5:').replace('socks4a:', 'socks4:');
return `${protocol}//${proxyUrl.host}`;
@ -95,163 +44,62 @@ const getProxyOptions = (currentProxy: ProxyState | null | undefined) => {
} satisfies Pick<LaunchOptions, 'proxy'>;
};
const COMMON_LAUNCH_ARGS = ['--no-sandbox', '--disable-setuid-sandbox', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list'];
// Patchright already patches playwright's default args (e.g. injects --disable-blink-features=AutomationControlled and strips --enable-automation), so we don't add those manually.
const getLaunchOptions = (currentProxy?: ProxyState | null): LaunchOptions => ({
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list'],
args: COMMON_LAUNCH_ARGS,
executablePath: config.chromiumExecutablePath || undefined,
headless: true,
...getProxyOptions(currentProxy),
});
// Browserless accepts launch options as a `launch` URL query parameter (URL-encoded JSON).
// (Patchright's own launch-server uses `launch-options` — RSSHub's WS_ENDPOINT targets browserless, so we emit `launch`.)
// The browserless schema also differs from patchright's LaunchOptions: no `executablePath`, and `ignoreHTTPSErrors` is renamed to `acceptInsecureCerts`.
type BrowserlessLaunchOptions = {
acceptInsecureCerts?: boolean;
args?: string[];
headless?: boolean;
ignoreDefaultArgs?: boolean | string[];
proxy?: LaunchOptions['proxy'];
slowMo?: number;
stealth?: boolean;
};
const toBrowserlessLaunchOptions = (currentProxy?: ProxyState | null): BrowserlessLaunchOptions => ({
acceptInsecureCerts: true,
args: COMMON_LAUNCH_ARGS,
headless: true,
stealth: true,
...getProxyOptions(currentProxy),
});
const getContextOptions = (): BrowserContextOptions => ({
ignoreHTTPSErrors: true,
userAgent: config.ua,
});
const createRouteRequest = (route: Route): HandledRouteRequest => {
const request = route.request();
const routeRequest = {
abort: async (errorCode) => {
routeRequest.handled = true;
await route.abort(errorCode);
},
continue: async (options) => {
routeRequest.handled = true;
await route.continue(options);
},
handled: false,
resourceType: () => request.resourceType(),
url: () => request.url(),
};
return routeRequest;
};
const runRequestHandlers = async (handlers: RequestHandler[], request: HandledRouteRequest, index = 0): Promise<void> => {
if (request.handled || index >= handlers.length) {
return;
}
await handlers[index](request);
await runRequestHandlers(handlers, request, index + 1);
};
const createFinishedRequest = (request: PlaywrightRequest, response: PlaywrightResponse | null): FinishedRequest => ({
response: () =>
response
? {
status: () => response.status(),
}
: null,
url: () => request.url(),
});
const patchPage = (page: PlaywrightPage, context: BrowserContext): Page => {
const compatPage = page as Page;
const requestHandlers: RequestHandler[] = [];
const originalGoto = page.goto.bind(page);
const originalOn = page.on.bind(page);
const originalRoute = page.route.bind(page);
const originalUnroute = page.unroute.bind(page);
let routeHandler: ((route: Route) => Promise<void>) | undefined;
let requestInterceptionEnabled = false;
compatPage.goto = (url, options) => originalGoto(url, normalizeGotoOptions(options));
compatPage.cookies = (urls) => context.cookies(urls);
compatPage.setCookie = async (...cookies) => {
await context.addCookies(cookies.map((cookie) => withDefaultCookiePath(cookie)));
};
compatPage.authenticate = async () => {};
compatPage.setUserAgent = async (userAgent) => {
const contextWithCDP = context as BrowserContext & {
newCDPSession?: (page: PlaywrightPage) => Promise<{
detach: () => Promise<void>;
send: (method: string, params?: Record<string, unknown>) => Promise<void>;
}>;
};
if (contextWithCDP.newCDPSession) {
const session = await contextWithCDP.newCDPSession(page);
await session.send('Network.setUserAgentOverride', { userAgent });
await session.detach();
return;
}
await page.setExtraHTTPHeaders({
'User-Agent': userAgent,
});
};
compatPage.setRequestInterception = async (enabled) => {
requestInterceptionEnabled = enabled;
if (enabled && !routeHandler) {
routeHandler = async (route) => {
const request = createRouteRequest(route);
await runRequestHandlers(requestHandlers, request);
if (request.handled) {
return;
}
await route.continue();
};
await originalRoute('**/*', routeHandler);
} else if (!enabled && routeHandler) {
await originalUnroute('**/*', routeHandler);
routeHandler = undefined;
}
};
compatPage.on = ((event: string, handler: (...args: any[]) => any) => {
if (event === 'request') {
requestHandlers.push(handler as RequestHandler);
if (!requestInterceptionEnabled) {
originalOn(event, handler);
}
return compatPage;
}
if (event === 'requestfinished') {
originalOn(event, async (request) => {
let response: PlaywrightResponse | null = null;
try {
response = await request.response();
} catch {
// The remote browser may close before Playwright resolves the response.
}
await (handler as RequestFinishedHandler)(createFinishedRequest(request, response));
});
return compatPage;
}
originalOn(event, handler);
return compatPage;
}) as Page['on'];
return compatPage;
};
const createCompatBrowser = async (browser: PlaywrightBrowser, contextOptions: BrowserContextOptions): Promise<Browser> => {
const context = await browser.newContext(contextOptions);
const compatBrowser = browser as Browser;
const originalClose = browser.close.bind(browser);
compatBrowser.newPage = async () => patchPage(await context.newPage(), context);
compatBrowser.setCookie = async (...cookies) => {
await context.addCookies(cookies.map((cookie) => withDefaultCookiePath(cookie)));
};
compatBrowser.cookies = (urls) => context.cookies(urls);
compatBrowser.userAgent = () => config.ua;
compatBrowser.close = async (options) => {
try {
await context.close();
} catch {
// Ignore already-closed contexts.
}
await originalClose(options);
};
return compatBrowser;
};
const launchBrowser = async (currentProxy?: ProxyState | null) => {
const launchOptions = getLaunchOptions(currentProxy);
const browser = config.playwrightWSEndpoint ? await chromium.connect(getEndpointWithLaunchOptions(config.playwrightWSEndpoint, launchOptions)) : await chromium.launch(launchOptions);
return createCompatBrowser(browser, getContextOptions());
const browser = config.playwrightWSEndpoint ? await chromium.connect(getBrowserlessEndpoint(config.playwrightWSEndpoint, toBrowserlessLaunchOptions(currentProxy))) : await chromium.launch(getLaunchOptions(currentProxy));
const context = await browser.newContext(getContextOptions());
return { browser, context };
};
const getEndpointWithLaunchOptions = (endpoint: string, launchOptions: LaunchOptions) => {
// Merge our launch options into the existing `launch` query parameter so endpoint-level options
// (e.g. `?launch=%7B%22stealth%22%3Atrue%7D`) are preserved instead of being overwritten.
const getBrowserlessEndpoint = (endpoint: string, launchOptions: BrowserlessLaunchOptions) => {
const endpointURL = new URL(endpoint);
endpointURL.searchParams.set('launch-options', JSON.stringify(launchOptions));
const existing = endpointURL.searchParams.get('launch');
let merged: BrowserlessLaunchOptions = launchOptions;
if (existing) {
try {
merged = { ...(JSON.parse(existing) as BrowserlessLaunchOptions), ...launchOptions };
} catch {
// Existing value is not JSON (could be base64 or malformed); leave caller's options as the source of truth.
}
}
endpointURL.searchParams.set('launch', JSON.stringify(merged));
return endpointURL.toString();
};
@ -262,13 +110,13 @@ const scheduleClose = (browser: Browser, timeout = 30000) => {
};
/**
* @returns Playwright browser
* @returns Playwright browser context (native `newPage()` shares state across calls)
*/
const outPlaywright = async () => {
const currentProxy = proxy.getCurrentProxy();
const browser = await launchBrowser(currentProxy && proxy.proxyObj.url_regex === '.*' ? currentProxy : null);
const { browser, context } = await launchBrowser(currentProxy && proxy.proxyObj.url_regex === '.*' ? currentProxy : null);
scheduleClose(browser);
return browser;
return context;
};
export default outPlaywright;
@ -285,7 +133,7 @@ export const getPlaywrightPage = async (
closeTimeout?: number;
gotoConfig?: GotoOptions;
noGoto?: boolean;
onBeforeLoad?: (page: Page, browser?: Browser) => Promise<void> | void;
onBeforeLoad?: (page: Page, context?: BrowserContext) => Promise<void> | void;
} = {}
) => {
let allowProxy = false;
@ -304,16 +152,16 @@ export const getPlaywrightPage = async (
const currentProxy = proxy.getCurrentProxy();
const currentProxyState = currentProxy && allowProxy ? currentProxy : null;
const hasProxy = Boolean(getProxyOptions(currentProxyState).proxy);
const browser = await launchBrowser(currentProxyState);
const { browser, context } = await launchBrowser(currentProxyState);
scheduleClose(browser, instanceOptions.closeTimeout);
const page = await browser.newPage();
const page = await context.newPage();
if (hasProxy && currentProxyState) {
logger.debug(`Proxying request in playwright via ${currentProxyState.uri}: ${url}`);
}
if (instanceOptions.onBeforeLoad) {
await instanceOptions.onBeforeLoad(page, browser);
await instanceOptions.onBeforeLoad(page, context);
}
if (!instanceOptions.noGoto) {
@ -330,12 +178,12 @@ export const getPlaywrightPage = async (
}
return {
browser,
context,
destroy: async () => {
await browser.close();
await context.close();
},
page,
};
};
export const getPuppeteerPage = getPlaywrightPage;
export { type Page } from 'patchright';

199
lib/utils/playwright.worker.ts
View File

@ -1,51 +1,10 @@
// Worker-compatible Playwright using Cloudflare Browser Run.
import type { Browser as PlaywrightBrowser, BrowserContext, Page as PlaywrightPage, Request as PlaywrightRequest, Response as PlaywrightResponse, Route } from '@cloudflare/playwright';
import type { Browser, Page } from '@cloudflare/playwright';
import { launch } from '@cloudflare/playwright';
import { config } from '@/config';
import logger from './logger';
type SetCookieParam = Parameters<BrowserContext['addCookies']>[0][number];
type Cookie = Awaited<ReturnType<BrowserContext['cookies']>>[number];
type GotoOptions = Parameters<PlaywrightPage['goto']>[1] & {
waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'networkidle0' | 'networkidle2';
};
type RouteRequest = {
abort: (errorCode?: string) => Promise<void>;
continue: (options?: Parameters<Route['continue']>[0]) => Promise<void>;
resourceType: () => ReturnType<PlaywrightRequest['resourceType']>;
url: () => string;
};
type FinishedRequest = {
response: () => {
status: () => number;
} | null;
url: () => string;
};
type RequestHandler = (request: RouteRequest) => Promise<void> | void;
type RequestFinishedHandler = (request: FinishedRequest) => Promise<void> | void;
type HandledRouteRequest = RouteRequest & { handled: boolean };
export type Page = PlaywrightPage & {
authenticate: (credentials: { password?: string; username?: string }) => Promise<void>;
cookies: (urls?: string | string[]) => Promise<Cookie[]>;
goto: (url: string, options?: GotoOptions) => ReturnType<PlaywrightPage['goto']>;
on: ((event: 'request', handler: RequestHandler) => Page) & ((event: 'requestfinished', handler: RequestFinishedHandler) => Page) & PlaywrightPage['on'];
setCookie: (...cookies: SetCookieParam[]) => Promise<void>;
setRequestInterception: (enabled: boolean) => Promise<void>;
setUserAgent: (userAgent: string) => Promise<void>;
};
export type Browser = PlaywrightBrowser & {
cookies: (urls?: string | string[]) => Promise<Cookie[]>;
newPage: () => Promise<Page>;
setCookie: (...cookies: SetCookieParam[]) => Promise<void>;
userAgent: () => string;
};
type GotoOptions = Parameters<Page['goto']>[1];
let browserBinding: any = null;
@ -60,142 +19,14 @@ const getBrowserBinding = () => {
return browserBinding;
};
const normalizeWaitUntil = (waitUntil: GotoOptions['waitUntil']) => (waitUntil === 'networkidle0' || waitUntil === 'networkidle2' ? 'networkidle' : waitUntil);
const normalizeGotoOptions = (options?: GotoOptions): Parameters<PlaywrightPage['goto']>[1] | undefined =>
options
? {
...options,
waitUntil: normalizeWaitUntil(options.waitUntil),
}
: options;
const withDefaultCookiePath = (cookie: SetCookieParam): SetCookieParam => ('domain' in cookie && !('path' in cookie) ? { ...cookie, path: '/' } : cookie);
const createRouteRequest = (route: Route): HandledRouteRequest => {
const request = route.request();
const routeRequest = {
abort: async (errorCode) => {
routeRequest.handled = true;
await route.abort(errorCode);
},
continue: async (options) => {
routeRequest.handled = true;
await route.continue(options);
},
handled: false,
resourceType: () => request.resourceType(),
url: () => request.url(),
};
return routeRequest;
};
const runRequestHandlers = async (handlers: RequestHandler[], request: HandledRouteRequest, index = 0): Promise<void> => {
if (request.handled || index >= handlers.length) {
return;
}
await handlers[index](request);
await runRequestHandlers(handlers, request, index + 1);
};
const createFinishedRequest = (request: PlaywrightRequest, response: PlaywrightResponse | null): FinishedRequest => ({
response: () =>
response
? {
status: () => response.status(),
}
: null,
url: () => request.url(),
});
const patchPage = (page: PlaywrightPage, context: BrowserContext): Page => {
const compatPage = page as Page;
const requestHandlers: RequestHandler[] = [];
const originalGoto = page.goto.bind(page);
const originalOn = page.on.bind(page);
const originalRoute = page.route.bind(page);
const originalUnroute = page.unroute.bind(page);
let routeHandler: ((route: Route) => Promise<void>) | undefined;
let requestInterceptionEnabled = false;
compatPage.goto = (url, options) => originalGoto(url, normalizeGotoOptions(options));
compatPage.cookies = (urls) => context.cookies(urls);
compatPage.setCookie = async (...cookies) => {
await context.addCookies(cookies.map((cookie) => withDefaultCookiePath(cookie)));
};
compatPage.authenticate = async () => {};
compatPage.setUserAgent = async (userAgent) => {
await page.setExtraHTTPHeaders({
'User-Agent': userAgent,
});
};
compatPage.setRequestInterception = async (enabled) => {
requestInterceptionEnabled = enabled;
if (enabled && !routeHandler) {
routeHandler = async (route) => {
const request = createRouteRequest(route);
await runRequestHandlers(requestHandlers, request);
if (request.handled) {
return;
}
await route.continue();
};
await originalRoute('**/*', routeHandler);
} else if (!enabled && routeHandler) {
await originalUnroute('**/*', routeHandler);
routeHandler = undefined;
}
};
compatPage.on = ((event: string, handler: (...args: any[]) => any) => {
if (event === 'request') {
requestHandlers.push(handler as RequestHandler);
if (!requestInterceptionEnabled) {
originalOn(event, handler);
}
return compatPage;
}
if (event === 'requestfinished') {
originalOn(event, async (request) => {
const response = await request.response();
await (handler as RequestFinishedHandler)(createFinishedRequest(request, response));
});
return compatPage;
}
originalOn(event, handler);
return compatPage;
}) as Page['on'];
return compatPage;
};
const createCompatBrowser = async (browser: PlaywrightBrowser): Promise<Browser> => {
const launchBrowser = async () => {
const browser = await launch(getBrowserBinding(), { keep_alive: 60000 });
const context = await browser.newContext({
ignoreHTTPSErrors: true,
});
const compatBrowser = browser as Browser;
const originalClose = browser.close.bind(browser);
compatBrowser.newPage = async () => patchPage(await context.newPage(), context);
compatBrowser.setCookie = async (...cookies) => {
await context.addCookies(cookies.map((cookie) => withDefaultCookiePath(cookie)));
};
compatBrowser.cookies = (urls) => context.cookies(urls);
compatBrowser.userAgent = () => config.ua;
compatBrowser.close = async (options) => {
try {
await context.close();
} catch {
// Ignore already-closed contexts.
}
await originalClose(options);
};
return compatBrowser;
return { browser, context };
};
const launchBrowser = async () => createCompatBrowser(await launch(getBrowserBinding(), { keep_alive: 60000 }));
const scheduleClose = (browser: Browser) => {
setTimeout(() => {
void browser.close();
@ -203,12 +34,12 @@ const scheduleClose = (browser: Browser) => {
};
/**
* @returns Playwright browser
* @returns Playwright browser context (native `newPage()` shares state across calls)
*/
const outPlaywright = async () => {
const browser = await launchBrowser();
const { browser, context } = await launchBrowser();
scheduleClose(browser);
return browser;
return context;
};
export default outPlaywright;
@ -221,17 +52,17 @@ export const getPlaywrightPage = async (
instanceOptions: {
gotoConfig?: GotoOptions;
noGoto?: boolean;
onBeforeLoad?: (page: Page, browser?: Browser) => Promise<void> | void;
onBeforeLoad?: (page: Page, context?: Awaited<ReturnType<typeof launchBrowser>>['context']) => Promise<void> | void;
} = {}
) => {
logger.debug(`Launching Cloudflare Browser for: ${url}`);
const browser = await launchBrowser();
const { browser, context } = await launchBrowser();
scheduleClose(browser);
const page = await browser.newPage();
const page = await context.newPage();
if (instanceOptions.onBeforeLoad) {
await instanceOptions.onBeforeLoad(page, browser);
await instanceOptions.onBeforeLoad(page, context);
}
if (!instanceOptions.noGoto) {
@ -244,12 +75,12 @@ export const getPlaywrightPage = async (
}
return {
browser,
context,
destroy: async () => {
await browser.close();
await context.close();
},
page,
};
};
export const getPuppeteerPage = getPlaywrightPage;
export { type Page } from '@cloudflare/playwright';

2
lib/utils/puppeteer.ts
View File

@ -1,2 +0,0 @@
export type { Browser, Page } from './playwright';
export { default, getPlaywrightPage, getPlaywrightPage as getPuppeteerPage, setBrowserBinding } from './playwright';

Some files were not shown because too many files have changed in this diff Show More