* Initial plan * Replace @tonyrl/rand-user-agent with header-generator and add sec-ch-* headers Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * Extract unwanted UA validation into standalone isValidUserAgent function Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * Add headerGeneratorPreset option to got and ofetch utilities * Replace randUserAgent usage in lib/routes with headerGeneratorPreset option Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * chore: pin header-generator to exact match * fix: replace randUserAgent with a static user agent string in config * Remove deprecated _randUserAgent function and related tests Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * refactor: rename to header-generator * Changes before error encountered Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * fix: update generateHeaders parameter type to Partial<HeaderGeneratorOptions> * fix: add ios preset * fix: rename headerGeneratorPreset to headerGeneratorOptions * feat: add MODERN_IOS preset and update header-generator patch hash * fix: use ios preset * fix: update FetchOptions to include headerGeneratorOptions type * refactor: simplify header generation and update header inclusion list * fix: remove header-generator patch * refactor: update user-agent handling in requests and tests * test: remove redundant headerGeneratorOptions test and add new ofetch header preset test * test: add custom user-agent test for ofetch * test: update user-agent expectation in ofetch headers test * Update lib/utils/ofetch.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor: replace hardcoded headers with generatedHeaders in fetch and get request handlers * refactor: rename test suite from rand-user-agent to header-generator and remove unused dependency * Update lib/utils/header-generator.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
91 lines
3.4 KiB
TypeScript
91 lines
3.4 KiB
TypeScript
import http from 'node:http';
|
|
import https from 'node:https';
|
|
import logger from '@/utils/logger';
|
|
import { config } from '@/config';
|
|
import proxy from '@/utils/proxy';
|
|
import { generateHeaders, generatedHeaders as HEADER_LIST } from '@/utils/header-generator';
|
|
import type { HeaderGeneratorOptions } from 'header-generator';
|
|
|
|
type Get = typeof http.get | typeof https.get | typeof http.request | typeof https.request;
|
|
|
|
interface ExtendedRequestOptions extends http.RequestOptions {
|
|
headerGeneratorOptions?: Partial<HeaderGeneratorOptions>;
|
|
}
|
|
|
|
const getWrappedGet: <T extends Get>(origin: T) => T = (origin) =>
|
|
function (this: any, ...args: Parameters<typeof origin>) {
|
|
let url: URL | null;
|
|
let options: ExtendedRequestOptions = {};
|
|
let callback: ((res: http.IncomingMessage) => void) | undefined;
|
|
if (typeof args[0] === 'string' || args[0] instanceof URL) {
|
|
url = new URL(args[0]);
|
|
if (typeof args[1] === 'object') {
|
|
options = args[1];
|
|
callback = args[2];
|
|
} else if (typeof args[1] === 'function') {
|
|
options = {};
|
|
callback = args[1];
|
|
}
|
|
} else {
|
|
options = args[0];
|
|
try {
|
|
url = new URL(options.href || `${options.protocol || 'http:'}//${options.hostname || options.host}${options.path}${options.search || (options.query ? `?${options.query}` : '')}`);
|
|
} catch {
|
|
url = null;
|
|
}
|
|
if (typeof args[1] === 'function') {
|
|
callback = args[1];
|
|
}
|
|
}
|
|
if (!url) {
|
|
return Reflect.apply(origin, this, args) as ReturnType<typeof origin>;
|
|
}
|
|
|
|
logger.debug(`Outgoing request: ${options.method || 'GET'} ${url}`);
|
|
|
|
options.headers = options.headers || {};
|
|
const headersLowerCaseKeys = new Set(Object.keys(options.headers).map((key) => key.toLowerCase()));
|
|
|
|
const generatedHeaders = generateHeaders(options.headerGeneratorOptions);
|
|
|
|
// ua
|
|
if (!headersLowerCaseKeys.has('user-agent')) {
|
|
options.headers['user-agent'] = config.ua;
|
|
}
|
|
|
|
for (const header of HEADER_LIST) {
|
|
if (!headersLowerCaseKeys.has(header) && generatedHeaders[header]) {
|
|
options.headers[header] = generatedHeaders[header];
|
|
}
|
|
}
|
|
|
|
// referer
|
|
if (!headersLowerCaseKeys.has('referer')) {
|
|
options.headers.referer = url.origin;
|
|
}
|
|
|
|
// proxy
|
|
if (!options.agent && proxy.agent) {
|
|
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
|
|
|
|
if (
|
|
proxyRegex.test(url.toString()) &&
|
|
url.protocol.startsWith('http') &&
|
|
url.host !== proxy.proxyUrlHandler?.host &&
|
|
url.host !== 'localhost' &&
|
|
!url.host.startsWith('127.') &&
|
|
!(config.puppeteerWSEndpoint?.includes(url.host) ?? false)
|
|
) {
|
|
options.agent = proxy.agent;
|
|
}
|
|
}
|
|
|
|
// Remove the headerGeneratorOptions before passing to the original function
|
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
const { headerGeneratorOptions, ...cleanOptions } = options;
|
|
|
|
return Reflect.apply(origin, this, [url, cleanOptions, callback]) as ReturnType<typeof origin>;
|
|
};
|
|
|
|
export default getWrappedGet;
|