Files
RSSHub/lib/utils/request-rewriter/get.ts
T
2ae10a7a02 feat: replace rand-user-agent with header-generator (#20480)
* Initial plan

* Replace @tonyrl/rand-user-agent with header-generator and add sec-ch-* headers

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* Extract unwanted UA validation into standalone isValidUserAgent function

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* Add headerGeneratorPreset option to got and ofetch utilities

* Replace randUserAgent usage in lib/routes with headerGeneratorPreset option

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* chore: pin header-generator to exact match

* fix: replace randUserAgent with a static user agent string in config

* Remove deprecated _randUserAgent function and related tests

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* refactor: rename to header-generator

* Changes before error encountered

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* fix: update generateHeaders parameter type to Partial<HeaderGeneratorOptions>

* fix: add ios preset

* fix: rename headerGeneratorPreset to headerGeneratorOptions

* feat: add MODERN_IOS preset and update header-generator patch hash

* fix: use ios preset

* fix: update FetchOptions to include headerGeneratorOptions type

* refactor: simplify header generation and update header inclusion list

* fix: remove header-generator patch

* refactor: update user-agent handling in requests and tests

* test: remove redundant headerGeneratorOptions test and add new ofetch header preset test

* test: add custom user-agent test for ofetch

* test: update user-agent expectation in ofetch headers test

* Update lib/utils/ofetch.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* refactor: replace hardcoded headers with generatedHeaders in fetch and get request handlers

* refactor: rename test suite from rand-user-agent to header-generator and remove unused dependency

* Update lib/utils/header-generator.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-11-14 00:49:41 +08:00

91 lines
3.4 KiB
TypeScript

import http from 'node:http';
import https from 'node:https';
import logger from '@/utils/logger';
import { config } from '@/config';
import proxy from '@/utils/proxy';
import { generateHeaders, generatedHeaders as HEADER_LIST } from '@/utils/header-generator';
import type { HeaderGeneratorOptions } from 'header-generator';
type Get = typeof http.get | typeof https.get | typeof http.request | typeof https.request;
interface ExtendedRequestOptions extends http.RequestOptions {
headerGeneratorOptions?: Partial<HeaderGeneratorOptions>;
}
const getWrappedGet: <T extends Get>(origin: T) => T = (origin) =>
function (this: any, ...args: Parameters<typeof origin>) {
let url: URL | null;
let options: ExtendedRequestOptions = {};
let callback: ((res: http.IncomingMessage) => void) | undefined;
if (typeof args[0] === 'string' || args[0] instanceof URL) {
url = new URL(args[0]);
if (typeof args[1] === 'object') {
options = args[1];
callback = args[2];
} else if (typeof args[1] === 'function') {
options = {};
callback = args[1];
}
} else {
options = args[0];
try {
url = new URL(options.href || `${options.protocol || 'http:'}//${options.hostname || options.host}${options.path}${options.search || (options.query ? `?${options.query}` : '')}`);
} catch {
url = null;
}
if (typeof args[1] === 'function') {
callback = args[1];
}
}
if (!url) {
return Reflect.apply(origin, this, args) as ReturnType<typeof origin>;
}
logger.debug(`Outgoing request: ${options.method || 'GET'} ${url}`);
options.headers = options.headers || {};
const headersLowerCaseKeys = new Set(Object.keys(options.headers).map((key) => key.toLowerCase()));
const generatedHeaders = generateHeaders(options.headerGeneratorOptions);
// ua
if (!headersLowerCaseKeys.has('user-agent')) {
options.headers['user-agent'] = config.ua;
}
for (const header of HEADER_LIST) {
if (!headersLowerCaseKeys.has(header) && generatedHeaders[header]) {
options.headers[header] = generatedHeaders[header];
}
}
// referer
if (!headersLowerCaseKeys.has('referer')) {
options.headers.referer = url.origin;
}
// proxy
if (!options.agent && proxy.agent) {
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
if (
proxyRegex.test(url.toString()) &&
url.protocol.startsWith('http') &&
url.host !== proxy.proxyUrlHandler?.host &&
url.host !== 'localhost' &&
!url.host.startsWith('127.') &&
!(config.puppeteerWSEndpoint?.includes(url.host) ?? false)
) {
options.agent = proxy.agent;
}
}
// Remove the headerGeneratorOptions before passing to the original function
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { headerGeneratorOptions, ...cleanOptions } = options;
return Reflect.apply(origin, this, [url, cleanOptions, callback]) as ReturnType<typeof origin>;
};
export default getWrappedGet;