import type { Browser as PlaywrightBrowser, BrowserContext, BrowserContextOptions, LaunchOptions, Page as PlaywrightPage, Request as PlaywrightRequest, Response as PlaywrightResponse, Route } from 'playwright'; import { chromium } from 'playwright'; import { config } from '@/config'; import logger from './logger'; import proxy from './proxy'; type SetCookieParam = Parameters[0][number]; type Cookie = Awaited>[number]; type GotoOptions = Parameters[1] & { waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'networkidle0' | 'networkidle2'; }; type ProxyState = NonNullable>; type RouteRequest = { abort: (errorCode?: string) => Promise; continue: (options?: Parameters[0]) => Promise; resourceType: () => ReturnType; url: () => string; }; type FinishedRequest = { response: () => { status: () => number; } | null; url: () => string; }; type RequestHandler = (request: RouteRequest) => Promise | void; type RequestFinishedHandler = (request: FinishedRequest) => Promise | void; type HandledRouteRequest = RouteRequest & { handled: boolean }; export type Page = PlaywrightPage & { authenticate: (credentials: { password?: string; username?: string }) => Promise; cookies: (urls?: string | string[]) => Promise; goto: (url: string, options?: GotoOptions) => ReturnType; on: ((event: 'request', handler: RequestHandler) => Page) & ((event: 'requestfinished', handler: RequestFinishedHandler) => Page) & PlaywrightPage['on']; setCookie: (...cookies: SetCookieParam[]) => Promise; setRequestInterception: (enabled: boolean) => Promise; setUserAgent: (userAgent: string) => Promise; }; export type Browser = PlaywrightBrowser & { cookies: (urls?: string | string[]) => Promise; newPage: () => Promise; setCookie: (...cookies: SetCookieParam[]) => Promise; userAgent: () => string; }; const normalizeWaitUntil = (waitUntil: GotoOptions['waitUntil']) => (waitUntil === 'networkidle0' || waitUntil === 'networkidle2' ? 'networkidle' : waitUntil); const normalizeGotoOptions = (options?: GotoOptions): Parameters[1] | undefined => options ? { ...options, waitUntil: normalizeWaitUntil(options.waitUntil), } : options; const withDefaultCookiePath = (cookie: SetCookieParam): SetCookieParam => ('domain' in cookie && !('path' in cookie) ? { ...cookie, path: '/' } : cookie); const proxyServerFromUrl = (proxyUrl: URL) => { const protocol = proxyUrl.protocol.replace('socks5h:', 'socks5:').replace('socks4a:', 'socks4:'); return `${protocol}//${proxyUrl.host}`; }; const getProxyOptions = (currentProxy: ProxyState | null | undefined) => { if (!currentProxy) { return {}; } const username = currentProxy.urlHandler?.username; const password = currentProxy.urlHandler?.password; if (username || password) { if (currentProxy.urlHandler.protocol !== 'http:') { logger.warn('SOCKS/HTTPS proxy with authentication is not supported by playwright, continue without proxy'); return {}; } return { proxy: { password: decodeURIComponent(password ?? ''), server: proxyServerFromUrl(currentProxy.urlHandler), username: decodeURIComponent(username ?? ''), }, } satisfies Pick; } return { proxy: { server: currentProxy.uri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://').replace(/\/$/, ''), }, } satisfies Pick; }; const getLaunchOptions = (currentProxy?: ProxyState | null): LaunchOptions => ({ args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list'], executablePath: config.chromiumExecutablePath || undefined, headless: true, ...getProxyOptions(currentProxy), }); const getContextOptions = (): BrowserContextOptions => ({ ignoreHTTPSErrors: true, }); const createRouteRequest = (route: Route): HandledRouteRequest => { const request = route.request(); const routeRequest = { abort: async (errorCode) => { routeRequest.handled = true; await route.abort(errorCode); }, continue: async (options) => { routeRequest.handled = true; await route.continue(options); }, handled: false, resourceType: () => request.resourceType(), url: () => request.url(), }; return routeRequest; }; const runRequestHandlers = async (handlers: RequestHandler[], request: HandledRouteRequest, index = 0): Promise => { if (request.handled || index >= handlers.length) { return; } await handlers[index](request); await runRequestHandlers(handlers, request, index + 1); }; const createFinishedRequest = (request: PlaywrightRequest, response: PlaywrightResponse | null): FinishedRequest => ({ response: () => response ? { status: () => response.status(), } : null, url: () => request.url(), }); const patchPage = (page: PlaywrightPage, context: BrowserContext): Page => { const compatPage = page as Page; const requestHandlers: RequestHandler[] = []; const originalGoto = page.goto.bind(page); const originalOn = page.on.bind(page); const originalRoute = page.route.bind(page); const originalUnroute = page.unroute.bind(page); let routeHandler: ((route: Route) => Promise) | undefined; let requestInterceptionEnabled = false; compatPage.goto = (url, options) => originalGoto(url, normalizeGotoOptions(options)); compatPage.cookies = (urls) => context.cookies(urls); compatPage.setCookie = async (...cookies) => { await context.addCookies(cookies.map((cookie) => withDefaultCookiePath(cookie))); }; compatPage.authenticate = async () => {}; compatPage.setUserAgent = async (userAgent) => { const contextWithCDP = context as BrowserContext & { newCDPSession?: (page: PlaywrightPage) => Promise<{ detach: () => Promise; send: (method: string, params?: Record) => Promise; }>; }; if (contextWithCDP.newCDPSession) { const session = await contextWithCDP.newCDPSession(page); await session.send('Network.setUserAgentOverride', { userAgent }); await session.detach(); return; } await page.setExtraHTTPHeaders({ 'User-Agent': userAgent, }); }; compatPage.setRequestInterception = async (enabled) => { requestInterceptionEnabled = enabled; if (enabled && !routeHandler) { routeHandler = async (route) => { const request = createRouteRequest(route); await runRequestHandlers(requestHandlers, request); if (request.handled) { return; } await route.continue(); }; await originalRoute('**/*', routeHandler); } else if (!enabled && routeHandler) { await originalUnroute('**/*', routeHandler); routeHandler = undefined; } }; compatPage.on = ((event: string, handler: (...args: any[]) => any) => { if (event === 'request') { requestHandlers.push(handler as RequestHandler); if (!requestInterceptionEnabled) { originalOn(event, handler); } return compatPage; } if (event === 'requestfinished') { originalOn(event, async (request) => { let response: PlaywrightResponse | null = null; try { response = await request.response(); } catch { // The remote browser may close before Playwright resolves the response. } await (handler as RequestFinishedHandler)(createFinishedRequest(request, response)); }); return compatPage; } originalOn(event, handler); return compatPage; }) as Page['on']; return compatPage; }; const createCompatBrowser = async (browser: PlaywrightBrowser, contextOptions: BrowserContextOptions): Promise => { const context = await browser.newContext(contextOptions); const compatBrowser = browser as Browser; const originalClose = browser.close.bind(browser); compatBrowser.newPage = async () => patchPage(await context.newPage(), context); compatBrowser.setCookie = async (...cookies) => { await context.addCookies(cookies.map((cookie) => withDefaultCookiePath(cookie))); }; compatBrowser.cookies = (urls) => context.cookies(urls); compatBrowser.userAgent = () => config.ua; compatBrowser.close = async (options) => { try { await context.close(); } catch { // Ignore already-closed contexts. } await originalClose(options); }; return compatBrowser; }; const launchBrowser = async (currentProxy?: ProxyState | null) => { const launchOptions = getLaunchOptions(currentProxy); const browser = config.playwrightWSEndpoint ? await chromium.connect(getEndpointWithLaunchOptions(config.playwrightWSEndpoint, launchOptions)) : await chromium.launch(launchOptions); return createCompatBrowser(browser, getContextOptions()); }; const getEndpointWithLaunchOptions = (endpoint: string, launchOptions: LaunchOptions) => { const endpointURL = new URL(endpoint); endpointURL.searchParams.set('launch-options', JSON.stringify(launchOptions)); return endpointURL.toString(); }; const scheduleClose = (browser: Browser, timeout = 30000) => { setTimeout(() => { void browser.close(); }, timeout); }; /** * @returns Playwright browser */ const outPlaywright = async () => { const currentProxy = proxy.getCurrentProxy(); const browser = await launchBrowser(currentProxy && proxy.proxyObj.url_regex === '.*' ? currentProxy : null); scheduleClose(browser); return browser; }; export default outPlaywright; // No-op in Node.js environment (used by Worker build via alias) export const setBrowserBinding = (_binding: any) => {}; /** * @returns Playwright page */ export const getPlaywrightPage = async ( url: string, instanceOptions: { closeTimeout?: number; gotoConfig?: GotoOptions; noGoto?: boolean; onBeforeLoad?: (page: Page, browser?: Browser) => Promise | void; } = {} ) => { let allowProxy = false; const proxyRegex = new RegExp(proxy.proxyObj.url_regex); let urlHandler: URL | undefined; try { urlHandler = new URL(url); } catch { // ignore invalid URLs such as about:blank } if (proxyRegex.test(url) && url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) { allowProxy = true; } const currentProxy = proxy.getCurrentProxy(); const currentProxyState = currentProxy && allowProxy ? currentProxy : null; const hasProxy = Boolean(getProxyOptions(currentProxyState).proxy); const browser = await launchBrowser(currentProxyState); scheduleClose(browser, instanceOptions.closeTimeout); const page = await browser.newPage(); if (hasProxy && currentProxyState) { logger.debug(`Proxying request in playwright via ${currentProxyState.uri}: ${url}`); } if (instanceOptions.onBeforeLoad) { await instanceOptions.onBeforeLoad(page, browser); } if (!instanceOptions.noGoto) { try { await page.goto(url, instanceOptions.gotoConfig || { waitUntil: 'domcontentloaded' }); } catch (error) { if (hasProxy && currentProxyState && proxy.multiProxy) { logger.warn(`Playwright navigation failed with proxy ${currentProxyState.uri}, marking as failed: ${error}`); proxy.markProxyFailed(currentProxyState.uri); throw error; } throw error; } } return { browser, destroy: async () => { await browser.close(); }, page, }; }; export const getPuppeteerPage = getPlaywrightPage;