From 4390cabb216e62a2132eeb51d2a8a68bb73abbcb Mon Sep 17 00:00:00 2001 From: Tony Date: Mon, 8 Jun 2026 10:31:39 +0800 Subject: [PATCH] feat(utils/playwright): add PLAYWRIGHT_CDP_ENDPOINT support (#22212) * feat(utils/playwright): add PLAYWRIGHT_CDP_ENDPOINT support * fix: close browser --- lib/config.ts | 3 + lib/utils/playwright.mock.test.ts | 199 ------------------- lib/utils/playwright.test.ts | 310 +++++++++++++++++++++++++++++- lib/utils/playwright.ts | 61 +++--- lib/utils/request-rewriter/get.ts | 2 +- 5 files changed, 337 insertions(+), 238 deletions(-) delete mode 100644 lib/utils/playwright.mock.test.ts diff --git a/lib/config.ts b/lib/config.ts index a460f3372..397ffb091 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -10,6 +10,7 @@ type ConfigEnvKeys = | 'NODE_NAME' | 'PLAYWRIGHT_WS_ENDPOINT' | 'PUPPETEER_WS_ENDPOINT' + | 'PLAYWRIGHT_CDP_ENDPOINT' | 'CHROMIUM_EXECUTABLE_PATH' // Network | 'PORT' @@ -260,6 +261,7 @@ export type Config = { isPackage: boolean; nodeName?: string; playwrightWSEndpoint?: string; + playwrightCDPEndpoint?: string; chromiumExecutablePath?: string; // network connect: { @@ -756,6 +758,7 @@ const calculateValue = () => { isPackage: !!envs.IS_PACKAGE, nodeName: envs.NODE_NAME, playwrightWSEndpoint: envs.PLAYWRIGHT_WS_ENDPOINT ?? envs.PUPPETEER_WS_ENDPOINT, + playwrightCDPEndpoint: envs.PLAYWRIGHT_CDP_ENDPOINT, chromiumExecutablePath: envs.CHROMIUM_EXECUTABLE_PATH, // network connect: { diff --git a/lib/utils/playwright.mock.test.ts b/lib/utils/playwright.mock.test.ts deleted file mode 100644 index 070d0bb95..000000000 --- a/lib/utils/playwright.mock.test.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; - -const connect = vi.fn(); -const connectOverCDP = vi.fn(); -const launch = vi.fn(); - -let page: any; -let context: any; -let browser: any; - -const createBrowserMocks = () => { - page = { - context: vi.fn(), - goto: vi.fn(), - on: vi.fn(), - setExtraHTTPHeaders: vi.fn(), - }; - - context = { - addCookies: vi.fn(), - close: vi.fn(() => Promise.resolve()), - cookies: vi.fn(), - newPage: vi.fn(() => Promise.resolve(page)), - }; - - browser = { - close: vi.fn(), - newContext: vi.fn(() => Promise.resolve(context)), - }; -}; - -const proxyMock = { - proxyObj: { url_regex: '.*' }, - proxyUrlHandler: new URL('http://proxy.local'), - multiProxy: undefined as any, - getCurrentProxy: vi.fn(), - markProxyFailed: vi.fn(), - getDispatcherForProxy: vi.fn(), -}; - -vi.mock('patchright', () => ({ - chromium: { - connect, - connectOverCDP, - launch, - }, -})); - -vi.mock('@/utils/proxy', () => ({ - default: proxyMock, -})); - -vi.mock('@/utils/logger', () => ({ - default: { - warn: vi.fn(), - debug: vi.fn(), - }, -})); - -const loadPlaywright = async () => { - vi.resetModules(); - const mod = await import('@/utils/playwright'); - return mod.getPlaywrightPage; -}; - -const resetMocks = () => { - createBrowserMocks(); - connect.mockReset(); - connectOverCDP.mockReset(); - launch.mockReset(); - proxyMock.multiProxy = undefined; - proxyMock.getCurrentProxy.mockReset(); - proxyMock.markProxyFailed.mockReset(); - delete process.env.PLAYWRIGHT_WS_ENDPOINT; - delete process.env.PUPPETEER_WS_ENDPOINT; -}; - -createBrowserMocks(); - -describe('getPlaywrightPage (mocked)', () => { - it('connects via ws endpoint and runs onBeforeLoad', async () => { - resetMocks(); - connect.mockResolvedValue(browser); - connectOverCDP.mockResolvedValue(browser); - launch.mockResolvedValue(browser); - page.goto.mockResolvedValue(undefined); - browser.close.mockResolvedValue(undefined); - process.env.PLAYWRIGHT_WS_ENDPOINT = 'ws://localhost:3000/?token=abc'; - proxyMock.getCurrentProxy.mockReturnValue(null); - - const getPlaywrightPage = await loadPlaywright(); - const onBeforeLoad = vi.fn(); - const contextClose = context.close; - const result = await getPlaywrightPage('https://example.com', { - noGoto: true, - onBeforeLoad, - }); - - const endpoint = connect.mock.calls[0][0] as string; - expect(connectOverCDP).not.toHaveBeenCalled(); - expect(endpoint).toContain('launch='); - expect(endpoint).not.toContain('launch-options='); - const launchOptions = JSON.parse(new URL(endpoint).searchParams.get('launch') || '{}'); - expect(launchOptions.args).not.toContainEqual(expect.stringContaining('--user-agent=')); - expect(launchOptions.executablePath).toBeUndefined(); - expect(launchOptions.acceptInsecureCerts).toBe(true); - expect(onBeforeLoad).toHaveBeenCalled(); - - await result.destroy(); - expect(contextClose).toHaveBeenCalled(); - }); - - it('merges browserless launch options with existing ws endpoint launch param', async () => { - resetMocks(); - connect.mockResolvedValue(browser); - launch.mockResolvedValue(browser); - page.goto.mockResolvedValue(undefined); - browser.close.mockResolvedValue(undefined); - process.env.PLAYWRIGHT_WS_ENDPOINT = `ws://localhost:3000/?token=abc&launch=${encodeURIComponent(JSON.stringify({ stealth: true }))}`; - proxyMock.getCurrentProxy.mockReturnValue(null); - - const getPlaywrightPage = await loadPlaywright(); - const result = await getPlaywrightPage('https://example.com', { noGoto: true }); - - const endpoint = connect.mock.calls[0][0] as string; - const launchOptions = JSON.parse(new URL(endpoint).searchParams.get('launch') || '{}'); - expect(launchOptions.stealth).toBe(true); - expect(launchOptions.headless).toBe(true); - - await result.destroy(); - }); - - it('does override the default HeadlessChrome user agent', async () => { - resetMocks(); - launch.mockResolvedValue(browser); - page.goto.mockResolvedValue(undefined); - proxyMock.getCurrentProxy.mockReturnValue(null); - - const getPlaywrightPage = await loadPlaywright(); - await getPlaywrightPage('https://example.com'); - - expect(launch).toHaveBeenCalledWith( - expect.objectContaining({ - args: expect.not.arrayContaining([expect.stringContaining('--user-agent=')]), - }) - ); - expect(browser.newContext).toHaveBeenCalledWith( - expect.objectContaining({ - userAgent: expect.any(String), - }) - ); - const contextOptions = browser.newContext.mock.calls[0][0]; - expect(contextOptions.userAgent).not.toMatch(/HeadlessChrome/i); - }); - - it('supports extending the browser auto close timeout', async () => { - vi.useFakeTimers(); - try { - resetMocks(); - launch.mockResolvedValue(browser); - browser.close.mockResolvedValue(undefined); - page.goto.mockResolvedValue(undefined); - proxyMock.getCurrentProxy.mockReturnValue(null); - - const getPlaywrightPage = await loadPlaywright(); - const close = browser.close; - await getPlaywrightPage('https://example.com', { - closeTimeout: 90000, - noGoto: true, - }); - - await vi.advanceTimersByTimeAsync(89999); - expect(close).not.toHaveBeenCalled(); - - await vi.advanceTimersByTimeAsync(1); - expect(close).toHaveBeenCalled(); - } finally { - vi.useRealTimers(); - } - }); - - it('marks proxy failed when navigation throws with multi-proxy', async () => { - resetMocks(); - launch.mockResolvedValue(browser); - page.goto.mockRejectedValueOnce(new Error('fail')); - - const currentProxy = { - uri: 'http://user:pass@proxy.local:8080', - urlHandler: new URL('http://user:pass@proxy.local:8080'), - }; - proxyMock.multiProxy = {}; - proxyMock.getCurrentProxy.mockReturnValue(currentProxy); - - const getPlaywrightPage = await loadPlaywright(); - await expect(getPlaywrightPage('https://example.com')).rejects.toThrow('fail'); - - expect(proxyMock.markProxyFailed).toHaveBeenCalledWith(currentProxy.uri); - }); -}); diff --git a/lib/utils/playwright.test.ts b/lib/utils/playwright.test.ts index b02de3ae8..95e3e47ce 100644 --- a/lib/utils/playwright.test.ts +++ b/lib/utils/playwright.test.ts @@ -3,6 +3,82 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import wait from './wait'; +const connect = vi.fn(); +const connectOverCDP = vi.fn(); +const launch = vi.fn(); + +let mockPage: any; +let mockContext: any; +let mockBrowser: any; + +const createBrowserMocks = () => { + mockPage = { + context: vi.fn(), + goto: vi.fn(), + on: vi.fn(), + setExtraHTTPHeaders: vi.fn(), + }; + + mockContext = { + addCookies: vi.fn(), + close: vi.fn(() => Promise.resolve()), + cookies: vi.fn(), + newPage: vi.fn(() => Promise.resolve(mockPage)), + }; + + mockBrowser = { + close: vi.fn(), + newContext: vi.fn(() => Promise.resolve(mockContext)), + }; +}; + +const proxyMock = { + proxyObj: { url_regex: '.*' }, + proxyUrlHandler: new URL('http://proxy.local'), + multiProxy: undefined as any, + getCurrentProxy: vi.fn(), + markProxyFailed: vi.fn(), + getDispatcherForProxy: vi.fn(), +}; + +createBrowserMocks(); + +const loadPlaywright = async () => { + vi.resetModules(); + vi.doMock('dotenv/config', () => ({})); + vi.doMock('patchright', () => ({ + chromium: { + connect, + connectOverCDP, + launch, + }, + })); + vi.doMock('@/utils/proxy', () => ({ + default: proxyMock, + })); + vi.doMock('@/utils/logger', () => ({ + default: { + warn: vi.fn(), + debug: vi.fn(), + }, + })); + const mod = await import('@/utils/playwright'); + return mod.getPlaywrightPage; +}; + +const resetMocks = () => { + createBrowserMocks(); + connect.mockReset(); + connectOverCDP.mockReset(); + launch.mockReset(); + proxyMock.multiProxy = undefined; + proxyMock.getCurrentProxy.mockReset(); + proxyMock.markProxyFailed.mockReset(); + delete process.env.PLAYWRIGHT_WS_ENDPOINT; + delete process.env.PUPPETEER_WS_ENDPOINT; + delete process.env.PLAYWRIGHT_CDP_ENDPOINT; +}; + let context: BrowserContext | null = null; afterEach(async () => { @@ -18,6 +94,11 @@ afterEach(async () => { delete process.env.PROXY_AUTH; delete process.env.PROXY_URL_REGEX; + vi.doUnmock('dotenv/config'); + vi.doUnmock('patchright'); + vi.doUnmock('@/utils/proxy'); + vi.doUnmock('@/utils/logger'); + vi.resetModules(); }); @@ -26,7 +107,6 @@ describe('playwright', () => { const { default: playwright } = await import('./playwright'); context = await playwright(); const browser = context.browser(); - const startTime = Date.now(); const page = await context.newPage(); await page.goto('https://www.google.com', { waitUntil: 'domcontentloaded', @@ -36,17 +116,11 @@ describe('playwright', () => { expect(html.length).toBeGreaterThan(0); expect(browser?.isConnected()).toBe(true); - const sleepTime = 31 * 1000 - (Date.now() - startTime); - if (sleepTime > 0) { - await wait(sleepTime); - } + await browser?.close(); expect(browser?.isConnected()).toBe(false); - context = null; - }, 45000); -}); + }, 10000); -describe('getPlaywrightPage', () => { - it('playwright run', async () => { + it('getPlaywrightPage', async () => { const { getPlaywrightPage } = await import('./playwright'); const playwright = await getPlaywrightPage('https://www.google.com'); const page = playwright.page; @@ -66,3 +140,219 @@ describe('getPlaywrightPage', () => { context = null; }, 45000); }); + +describe('getPlaywrightPage (mocked)', () => { + it('connects via ws endpoint and runs onBeforeLoad', async () => { + resetMocks(); + connect.mockResolvedValue(mockBrowser); + connectOverCDP.mockResolvedValue(mockBrowser); + launch.mockResolvedValue(mockBrowser); + mockPage.goto.mockResolvedValue(undefined); + mockBrowser.close.mockResolvedValue(undefined); + process.env.PLAYWRIGHT_WS_ENDPOINT = 'ws://localhost:3000/?token=abc'; + proxyMock.getCurrentProxy.mockReturnValue(null); + + const getPlaywrightPage = await loadPlaywright(); + const onBeforeLoad = vi.fn(); + const contextClose = mockContext.close; + const result = await getPlaywrightPage('https://example.com', { + noGoto: true, + onBeforeLoad, + }); + + const endpoint = connect.mock.calls[0][0] as string; + expect(connectOverCDP).not.toHaveBeenCalled(); + expect(endpoint).toContain('launch='); + expect(endpoint).not.toContain('launch-options='); + const launchOptions = JSON.parse(new URL(endpoint).searchParams.get('launch') || '{}'); + expect(launchOptions.args).not.toContainEqual(expect.stringContaining('--user-agent=')); + expect(launchOptions.executablePath).toBeUndefined(); + expect(launchOptions.ignoreHTTPSErrors).toBeUndefined(); + expect(launchOptions.stealth).toBeUndefined(); + expect(launchOptions.headless).toBe(true); + expect(onBeforeLoad).toHaveBeenCalled(); + + await result.destroy(); + expect(contextClose).toHaveBeenCalled(); + }); + + it('overrides an existing ws endpoint launch param, dropping invalid keys like stealth', async () => { + resetMocks(); + connect.mockResolvedValue(mockBrowser); + launch.mockResolvedValue(mockBrowser); + mockPage.goto.mockResolvedValue(undefined); + mockBrowser.close.mockResolvedValue(undefined); + process.env.PLAYWRIGHT_WS_ENDPOINT = `ws://localhost:3000/?token=abc&launch=${encodeURIComponent(JSON.stringify({ stealth: true }))}`; + proxyMock.getCurrentProxy.mockReturnValue(null); + + const getPlaywrightPage = await loadPlaywright(); + const result = await getPlaywrightPage('https://example.com', { noGoto: true }); + + const endpoint = connect.mock.calls[0][0] as string; + const launchOptions = JSON.parse(new URL(endpoint).searchParams.get('launch') || '{}'); + expect(launchOptions.stealth).toBeUndefined(); + expect(launchOptions.headless).toBe(true); + expect(Array.isArray(launchOptions.args)).toBe(true); + + await result.destroy(); + }); + + // Playwright WS uses `proxy` object (w/auth), while CDP uses `--proxy-server` (w/o auth). + it.each([ + { + name: 'WS route sends an unauthenticated proxy as a Playwright proxy object', + route: 'ws', + endpointVar: 'PLAYWRIGHT_WS_ENDPOINT', + endpoint: 'ws://localhost:3000/?token=abc', + proxyUri: 'http://proxy.local:8080', + assertLaunch: (lo: any) => { + expect(lo.proxy).toEqual({ server: 'http://proxy.local:8080' }); + expect(lo.args).not.toContainEqual(expect.stringContaining('--proxy-server=')); + }, + }, + { + name: 'WS route sends an authenticated http proxy with credentials', + route: 'ws', + endpointVar: 'PLAYWRIGHT_WS_ENDPOINT', + endpoint: 'ws://localhost:3000/?token=abc', + proxyUri: 'http://user:pass@proxy.local:8080', + assertLaunch: (lo: any) => { + expect(lo.proxy).toEqual({ server: 'http://proxy.local:8080', username: 'user', password: 'pass' }); + }, + }, + { + name: 'CDP route sends an unauthenticated proxy via the --proxy-server arg', + route: 'cdp', + endpointVar: 'PLAYWRIGHT_CDP_ENDPOINT', + endpoint: 'ws://localhost:3000/chromium?token=abc', + proxyUri: 'http://proxy.local:8080', + assertLaunch: (lo: any) => { + expect(lo.proxy).toBeUndefined(); + expect(lo.args).toContain('--proxy-server=http://proxy.local:8080'); + expect(lo.stealth).toBe(true); + }, + }, + { + name: 'CDP route skips an authenticated proxy (--proxy-server cannot carry credentials)', + route: 'cdp', + endpointVar: 'PLAYWRIGHT_CDP_ENDPOINT', + endpoint: 'ws://localhost:3000/chromium?token=abc', + proxyUri: 'http://user:pass@proxy.local:8080', + assertLaunch: (lo: any) => { + expect(lo.proxy).toBeUndefined(); + expect(lo.args).not.toContainEqual(expect.stringContaining('--proxy-server=')); + }, + }, + ])('$name', async ({ route, endpointVar, endpoint, proxyUri, assertLaunch }) => { + resetMocks(); + connect.mockResolvedValue(mockBrowser); + connectOverCDP.mockResolvedValue(mockBrowser); + launch.mockResolvedValue(mockBrowser); + mockPage.goto.mockResolvedValue(undefined); + mockBrowser.close.mockResolvedValue(undefined); + process.env[endpointVar] = endpoint; + proxyMock.getCurrentProxy.mockReturnValue({ uri: proxyUri, urlHandler: new URL(proxyUri) }); + + const getPlaywrightPage = await loadPlaywright(); + const result = await getPlaywrightPage('https://example.com', { noGoto: true }); + + const connectMock = route === 'cdp' ? connectOverCDP : connect; + const endpointUrl = connectMock.mock.calls[0][0] as string; + const launchOptions = JSON.parse(new URL(endpointUrl).searchParams.get('launch') || '{}'); + assertLaunch(launchOptions); + + await result.destroy(); + }); + + it('connects via connectOverCDP with stealth, taking priority over the WS endpoint when both are set', async () => { + resetMocks(); + connect.mockResolvedValue(mockBrowser); + connectOverCDP.mockResolvedValue(mockBrowser); + launch.mockResolvedValue(mockBrowser); + mockPage.goto.mockResolvedValue(undefined); + mockBrowser.close.mockResolvedValue(undefined); + process.env.PLAYWRIGHT_WS_ENDPOINT = 'ws://localhost:3000/chromium/playwright?token=abc'; + process.env.PLAYWRIGHT_CDP_ENDPOINT = 'ws://localhost:3000/chromium?token=abc'; + proxyMock.getCurrentProxy.mockReturnValue(null); + + const getPlaywrightPage = await loadPlaywright(); + const result = await getPlaywrightPage('https://example.com', { noGoto: true }); + + expect(connectOverCDP).toHaveBeenCalled(); + expect(connect).not.toHaveBeenCalled(); + expect(launch).not.toHaveBeenCalled(); + const endpoint = connectOverCDP.mock.calls[0][0] as string; + const launchOptions = JSON.parse(new URL(endpoint).searchParams.get('launch') || '{}'); + expect(launchOptions.stealth).toBe(true); + expect(launchOptions.headless).toBe(true); + + await result.destroy(); + }); + + it('does override the default HeadlessChrome user agent', async () => { + resetMocks(); + launch.mockResolvedValue(mockBrowser); + mockPage.goto.mockResolvedValue(undefined); + proxyMock.getCurrentProxy.mockReturnValue(null); + + const getPlaywrightPage = await loadPlaywright(); + await getPlaywrightPage('https://example.com'); + + expect(launch).toHaveBeenCalledWith( + expect.objectContaining({ + args: expect.not.arrayContaining([expect.stringContaining('--user-agent=')]), + }) + ); + expect(mockBrowser.newContext).toHaveBeenCalledWith( + expect.objectContaining({ + userAgent: expect.any(String), + }) + ); + const contextOptions = mockBrowser.newContext.mock.calls[0][0]; + expect(contextOptions.userAgent).not.toMatch(/HeadlessChrome/i); + }); + + it('supports extending the browser auto close timeout', async () => { + vi.useFakeTimers(); + try { + resetMocks(); + launch.mockResolvedValue(mockBrowser); + mockBrowser.close.mockResolvedValue(undefined); + mockPage.goto.mockResolvedValue(undefined); + proxyMock.getCurrentProxy.mockReturnValue(null); + + const getPlaywrightPage = await loadPlaywright(); + const close = mockBrowser.close; + await getPlaywrightPage('https://example.com', { + closeTimeout: 90000, + noGoto: true, + }); + + await vi.advanceTimersByTimeAsync(89999); + expect(close).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1); + expect(close).toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it('marks proxy failed when navigation throws with multi-proxy', async () => { + resetMocks(); + launch.mockResolvedValue(mockBrowser); + mockPage.goto.mockRejectedValueOnce(new Error('fail')); + + const currentProxy = { + uri: 'http://user:pass@proxy.local:8080', + urlHandler: new URL('http://user:pass@proxy.local:8080'), + }; + proxyMock.multiProxy = {}; + proxyMock.getCurrentProxy.mockReturnValue(currentProxy); + + const getPlaywrightPage = await loadPlaywright(); + await expect(getPlaywrightPage('https://example.com')).rejects.toThrow('fail'); + + expect(proxyMock.markProxyFailed).toHaveBeenCalledWith(currentProxy.uri); + }); +}); diff --git a/lib/utils/playwright.ts b/lib/utils/playwright.ts index 2ed7178b2..7740c70ca 100644 --- a/lib/utils/playwright.ts +++ b/lib/utils/playwright.ts @@ -44,9 +44,9 @@ const getProxyOptions = (currentProxy: ProxyState | null | undefined) => { } satisfies Pick; }; +// Patchright already patches playwright's default args (e.g. injects --disable-blink-features=AutomationControlled and strips --enable-automation) const COMMON_LAUNCH_ARGS = ['--no-sandbox', '--disable-setuid-sandbox', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list']; -// Patchright already patches playwright's default args (e.g. injects --disable-blink-features=AutomationControlled and strips --enable-automation), so we don't add those manually. const getLaunchOptions = (currentProxy?: ProxyState | null): LaunchOptions => ({ args: COMMON_LAUNCH_ARGS, executablePath: config.chromiumExecutablePath || undefined, @@ -54,52 +54,57 @@ const getLaunchOptions = (currentProxy?: ProxyState | null): LaunchOptions => ({ ...getProxyOptions(currentProxy), }); -// Browserless accepts launch options as a `launch` URL query parameter (URL-encoded JSON). -// (Patchright's own launch-server uses `launch-options` — RSSHub's WS_ENDPOINT targets browserless, so we emit `launch`.) -// The browserless schema also differs from patchright's LaunchOptions: no `executablePath`, and `ignoreHTTPSErrors` is renamed to `acceptInsecureCerts`. -type BrowserlessLaunchOptions = { - acceptInsecureCerts?: boolean; - args?: string[]; - headless?: boolean; - ignoreDefaultArgs?: boolean | string[]; - proxy?: LaunchOptions['proxy']; - slowMo?: number; - stealth?: boolean; -}; +type BrowserlessLaunchOptions = Pick; +type BrowserlessCdpLaunchOptions = Omit & { stealth?: boolean }; const toBrowserlessLaunchOptions = (currentProxy?: ProxyState | null): BrowserlessLaunchOptions => ({ - acceptInsecureCerts: true, args: COMMON_LAUNCH_ARGS, headless: true, - stealth: true, ...getProxyOptions(currentProxy), }); +// CDP accepts `stealth` but NOT a `proxy` object. +const toBrowserlessCDPLaunchOptions = (currentProxy?: ProxyState | null): BrowserlessCdpLaunchOptions => { + let proxyServerArgs: string[] = []; + + if (currentProxy) { + if (currentProxy.urlHandler?.username || currentProxy.urlHandler?.password) { + logger.warn('Proxy authentication is not supported over CDP (--proxy-server), continue without proxy'); + } else { + const server = currentProxy.uri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://').replace(/\/$/, ''); + proxyServerArgs = [`--proxy-server=${server}`]; + } + } + + return { + args: [...COMMON_LAUNCH_ARGS, ...proxyServerArgs], + headless: true, + stealth: true, + }; +}; + const getContextOptions = (): BrowserContextOptions => ({ ignoreHTTPSErrors: true, userAgent: config.ua, }); +// CDP > WS > local const launchBrowser = async (currentProxy?: ProxyState | null) => { - const browser = config.playwrightWSEndpoint ? await chromium.connect(getBrowserlessEndpoint(config.playwrightWSEndpoint, toBrowserlessLaunchOptions(currentProxy))) : await chromium.launch(getLaunchOptions(currentProxy)); + let browser: Browser; + if (config.playwrightCDPEndpoint) { + browser = await chromium.connectOverCDP(getBrowserlessEndpoint(config.playwrightCDPEndpoint, toBrowserlessCDPLaunchOptions(currentProxy))); + } else if (config.playwrightWSEndpoint) { + browser = await chromium.connect(getBrowserlessEndpoint(config.playwrightWSEndpoint, toBrowserlessLaunchOptions(currentProxy))); + } else { + browser = await chromium.launch(getLaunchOptions(currentProxy)); + } const context = await browser.newContext(getContextOptions()); return { browser, context }; }; -// Merge our launch options into the existing `launch` query parameter so endpoint-level options -// (e.g. `?launch=%7B%22stealth%22%3Atrue%7D`) are preserved instead of being overwritten. const getBrowserlessEndpoint = (endpoint: string, launchOptions: BrowserlessLaunchOptions) => { const endpointURL = new URL(endpoint); - const existing = endpointURL.searchParams.get('launch'); - let merged: BrowserlessLaunchOptions = launchOptions; - if (existing) { - try { - merged = { ...(JSON.parse(existing) as BrowserlessLaunchOptions), ...launchOptions }; - } catch { - // Existing value is not JSON (could be base64 or malformed); leave caller's options as the source of truth. - } - } - endpointURL.searchParams.set('launch', JSON.stringify(merged)); + endpointURL.searchParams.set('launch', JSON.stringify(launchOptions)); return endpointURL.toString(); }; diff --git a/lib/utils/request-rewriter/get.ts b/lib/utils/request-rewriter/get.ts index 25cc7a117..b7b9b8e80 100644 --- a/lib/utils/request-rewriter/get.ts +++ b/lib/utils/request-rewriter/get.ts @@ -80,7 +80,7 @@ const getWrappedGet: (origin: T) => T = (origin) => url.host !== proxy.proxyUrlHandler?.host && url.host !== 'localhost' && !url.host.startsWith('127.') && - !(config.playwrightWSEndpoint?.includes(url.host) ?? false) + ![config.playwrightWSEndpoint, config.playwrightCDPEndpoint].some((endpoint) => endpoint?.includes(url.host)) ) { options.agent = proxy.agent; }