* Initial plan * Replace @tonyrl/rand-user-agent with header-generator and add sec-ch-* headers Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * Extract unwanted UA validation into standalone isValidUserAgent function Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * Add headerGeneratorPreset option to got and ofetch utilities * Replace randUserAgent usage in lib/routes with headerGeneratorPreset option Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * chore: pin header-generator to exact match * fix: replace randUserAgent with a static user agent string in config * Remove deprecated _randUserAgent function and related tests Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * refactor: rename to header-generator * Changes before error encountered Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * fix: update generateHeaders parameter type to Partial<HeaderGeneratorOptions> * fix: add ios preset * fix: rename headerGeneratorPreset to headerGeneratorOptions * feat: add MODERN_IOS preset and update header-generator patch hash * fix: use ios preset * fix: update FetchOptions to include headerGeneratorOptions type * refactor: simplify header generation and update header inclusion list * fix: remove header-generator patch * refactor: update user-agent handling in requests and tests * test: remove redundant headerGeneratorOptions test and add new ofetch header preset test * test: add custom user-agent test for ofetch * test: update user-agent expectation in ofetch headers test * Update lib/utils/ofetch.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor: replace hardcoded headers with generatedHeaders in fetch and get request handlers * refactor: rename test suite from rand-user-agent to header-generator and remove unused dependency * Update lib/utils/header-generator.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
126 lines
4.4 KiB
TypeScript
126 lines
4.4 KiB
TypeScript
import logger from '@/utils/logger';
|
|
import { config } from '@/config';
|
|
import undici, { Request, RequestInfo, RequestInit } from 'undici';
|
|
import proxy from '@/utils/proxy';
|
|
import { RateLimiterMemory, RateLimiterQueue } from 'rate-limiter-flexible';
|
|
import { useRegisterRequest } from 'node-network-devtools';
|
|
import { generateHeaders, generatedHeaders as HEADER_LIST } from '@/utils/header-generator';
|
|
import type { HeaderGeneratorOptions } from 'header-generator';
|
|
|
|
const limiter = new RateLimiterMemory({
|
|
points: 10,
|
|
duration: 1,
|
|
execEvenly: true,
|
|
});
|
|
|
|
const limiterQueue = new RateLimiterQueue(limiter, {
|
|
maxQueueSize: 4800,
|
|
});
|
|
|
|
export const useCustomHeader = (headers: Headers) => {
|
|
process.env.NODE_ENV === 'dev' &&
|
|
useRegisterRequest((req) => {
|
|
for (const [key, value] of headers.entries()) {
|
|
req.requestHeaders[key] = value;
|
|
}
|
|
return req;
|
|
});
|
|
};
|
|
|
|
const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: RequestInit & { headerGeneratorOptions?: Partial<HeaderGeneratorOptions> }) => {
|
|
const request = new Request(input, init);
|
|
const options: RequestInit = {};
|
|
|
|
logger.debug(`Outgoing request: ${request.method} ${request.url}`);
|
|
|
|
const generatedHeaders = generateHeaders(init?.headerGeneratorOptions);
|
|
|
|
// ua
|
|
if (!request.headers.has('user-agent')) {
|
|
request.headers.set('user-agent', config.ua);
|
|
}
|
|
|
|
for (const header of HEADER_LIST) {
|
|
if (!request.headers.has(header) && generatedHeaders[header]) {
|
|
request.headers.set(header, generatedHeaders[header]);
|
|
}
|
|
}
|
|
|
|
// referer
|
|
if (!request.headers.get('referer')) {
|
|
try {
|
|
const urlHandler = new URL(request.url);
|
|
request.headers.set('referer', urlHandler.origin);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
|
|
let isRetry = false;
|
|
if (request.headers.get('x-prefer-proxy')) {
|
|
isRetry = true;
|
|
request.headers.delete('x-prefer-proxy');
|
|
}
|
|
|
|
config.enableRemoteDebugging && useCustomHeader(request.headers);
|
|
|
|
// proxy
|
|
if (!init?.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) {
|
|
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
|
|
let urlHandler;
|
|
try {
|
|
urlHandler = new URL(request.url);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
if (proxyRegex.test(request.url) && request.url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) {
|
|
const currentProxy = proxy.getCurrentProxy();
|
|
if (currentProxy) {
|
|
const dispatcher = proxy.getDispatcherForProxy(currentProxy);
|
|
if (dispatcher) {
|
|
options.dispatcher = dispatcher;
|
|
logger.debug(`Proxying request via ${currentProxy.uri}: ${request.url}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
await limiterQueue.removeTokens(1);
|
|
|
|
const maxRetries = proxy.multiProxy?.allProxies.length || 1;
|
|
|
|
const attemptRequest = async (attempt: number): Promise<Response> => {
|
|
try {
|
|
return await undici.fetch(request, options);
|
|
} catch (error) {
|
|
if (options.dispatcher && proxy.multiProxy && attempt < maxRetries - 1) {
|
|
const currentProxy = proxy.getCurrentProxy();
|
|
if (currentProxy) {
|
|
logger.warn(`Request failed with proxy ${currentProxy.uri}, trying next proxy: ${error}`);
|
|
proxy.markProxyFailed(currentProxy.uri);
|
|
|
|
const nextProxy = proxy.getCurrentProxy();
|
|
if (nextProxy && nextProxy.uri !== currentProxy.uri) {
|
|
const nextDispatcher = proxy.getDispatcherForProxy(nextProxy);
|
|
if (nextDispatcher) {
|
|
options.dispatcher = nextDispatcher;
|
|
}
|
|
logger.debug(`Retrying request with proxy ${nextProxy.uri}: ${request.url}`);
|
|
return attemptRequest(attempt + 1);
|
|
} else {
|
|
logger.warn('No more proxies available, trying without proxy');
|
|
delete options.dispatcher;
|
|
return attemptRequest(attempt + 1);
|
|
}
|
|
}
|
|
}
|
|
throw error;
|
|
}
|
|
};
|
|
|
|
return attemptRequest(0);
|
|
};
|
|
|
|
export default wrappedFetch;
|