Files
RSSHub/lib/utils/request-rewriter/fetch.ts
T
2ae10a7a02 feat: replace rand-user-agent with header-generator (#20480)
* Initial plan

* Replace @tonyrl/rand-user-agent with header-generator and add sec-ch-* headers

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* Extract unwanted UA validation into standalone isValidUserAgent function

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* Add headerGeneratorPreset option to got and ofetch utilities

* Replace randUserAgent usage in lib/routes with headerGeneratorPreset option

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* chore: pin header-generator to exact match

* fix: replace randUserAgent with a static user agent string in config

* Remove deprecated _randUserAgent function and related tests

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* refactor: rename to header-generator

* Changes before error encountered

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* fix: update generateHeaders parameter type to Partial<HeaderGeneratorOptions>

* fix: add ios preset

* fix: rename headerGeneratorPreset to headerGeneratorOptions

* feat: add MODERN_IOS preset and update header-generator patch hash

* fix: use ios preset

* fix: update FetchOptions to include headerGeneratorOptions type

* refactor: simplify header generation and update header inclusion list

* fix: remove header-generator patch

* refactor: update user-agent handling in requests and tests

* test: remove redundant headerGeneratorOptions test and add new ofetch header preset test

* test: add custom user-agent test for ofetch

* test: update user-agent expectation in ofetch headers test

* Update lib/utils/ofetch.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* refactor: replace hardcoded headers with generatedHeaders in fetch and get request handlers

* refactor: rename test suite from rand-user-agent to header-generator and remove unused dependency

* Update lib/utils/header-generator.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-11-14 00:49:41 +08:00

126 lines
4.4 KiB
TypeScript

import logger from '@/utils/logger';
import { config } from '@/config';
import undici, { Request, RequestInfo, RequestInit } from 'undici';
import proxy from '@/utils/proxy';
import { RateLimiterMemory, RateLimiterQueue } from 'rate-limiter-flexible';
import { useRegisterRequest } from 'node-network-devtools';
import { generateHeaders, generatedHeaders as HEADER_LIST } from '@/utils/header-generator';
import type { HeaderGeneratorOptions } from 'header-generator';
const limiter = new RateLimiterMemory({
points: 10,
duration: 1,
execEvenly: true,
});
const limiterQueue = new RateLimiterQueue(limiter, {
maxQueueSize: 4800,
});
export const useCustomHeader = (headers: Headers) => {
process.env.NODE_ENV === 'dev' &&
useRegisterRequest((req) => {
for (const [key, value] of headers.entries()) {
req.requestHeaders[key] = value;
}
return req;
});
};
const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: RequestInit & { headerGeneratorOptions?: Partial<HeaderGeneratorOptions> }) => {
const request = new Request(input, init);
const options: RequestInit = {};
logger.debug(`Outgoing request: ${request.method} ${request.url}`);
const generatedHeaders = generateHeaders(init?.headerGeneratorOptions);
// ua
if (!request.headers.has('user-agent')) {
request.headers.set('user-agent', config.ua);
}
for (const header of HEADER_LIST) {
if (!request.headers.has(header) && generatedHeaders[header]) {
request.headers.set(header, generatedHeaders[header]);
}
}
// referer
if (!request.headers.get('referer')) {
try {
const urlHandler = new URL(request.url);
request.headers.set('referer', urlHandler.origin);
} catch {
// ignore
}
}
let isRetry = false;
if (request.headers.get('x-prefer-proxy')) {
isRetry = true;
request.headers.delete('x-prefer-proxy');
}
config.enableRemoteDebugging && useCustomHeader(request.headers);
// proxy
if (!init?.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) {
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
let urlHandler;
try {
urlHandler = new URL(request.url);
} catch {
// ignore
}
if (proxyRegex.test(request.url) && request.url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) {
const currentProxy = proxy.getCurrentProxy();
if (currentProxy) {
const dispatcher = proxy.getDispatcherForProxy(currentProxy);
if (dispatcher) {
options.dispatcher = dispatcher;
logger.debug(`Proxying request via ${currentProxy.uri}: ${request.url}`);
}
}
}
}
await limiterQueue.removeTokens(1);
const maxRetries = proxy.multiProxy?.allProxies.length || 1;
const attemptRequest = async (attempt: number): Promise<Response> => {
try {
return await undici.fetch(request, options);
} catch (error) {
if (options.dispatcher && proxy.multiProxy && attempt < maxRetries - 1) {
const currentProxy = proxy.getCurrentProxy();
if (currentProxy) {
logger.warn(`Request failed with proxy ${currentProxy.uri}, trying next proxy: ${error}`);
proxy.markProxyFailed(currentProxy.uri);
const nextProxy = proxy.getCurrentProxy();
if (nextProxy && nextProxy.uri !== currentProxy.uri) {
const nextDispatcher = proxy.getDispatcherForProxy(nextProxy);
if (nextDispatcher) {
options.dispatcher = nextDispatcher;
}
logger.debug(`Retrying request with proxy ${nextProxy.uri}: ${request.url}`);
return attemptRequest(attempt + 1);
} else {
logger.warn('No more proxies available, trying without proxy');
delete options.dispatcher;
return attemptRequest(attempt + 1);
}
}
}
throw error;
}
};
return attemptRequest(0);
};
export default wrappedFetch;