Files
RSSHub/lib/utils/header-generator.ts
T
2ae10a7a02 feat: replace rand-user-agent with header-generator (#20480)
* Initial plan

* Replace @tonyrl/rand-user-agent with header-generator and add sec-ch-* headers

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* Extract unwanted UA validation into standalone isValidUserAgent function

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* Add headerGeneratorPreset option to got and ofetch utilities

* Replace randUserAgent usage in lib/routes with headerGeneratorPreset option

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* chore: pin header-generator to exact match

* fix: replace randUserAgent with a static user agent string in config

* Remove deprecated _randUserAgent function and related tests

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* refactor: rename to header-generator

* Changes before error encountered

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* fix: update generateHeaders parameter type to Partial<HeaderGeneratorOptions>

* fix: add ios preset

* fix: rename headerGeneratorPreset to headerGeneratorOptions

* feat: add MODERN_IOS preset and update header-generator patch hash

* fix: use ios preset

* fix: update FetchOptions to include headerGeneratorOptions type

* refactor: simplify header generation and update header inclusion list

* fix: remove header-generator patch

* refactor: update user-agent handling in requests and tests

* test: remove redundant headerGeneratorOptions test and add new ofetch header preset test

* test: add custom user-agent test for ofetch

* test: update user-agent expectation in ofetch headers test

* Update lib/utils/ofetch.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* refactor: replace hardcoded headers with generatedHeaders in fetch and get request handlers

* refactor: rename test suite from rand-user-agent to header-generator and remove unused dependency

* Update lib/utils/header-generator.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-11-14 00:49:41 +08:00

87 lines
2.8 KiB
TypeScript

import { HeaderGenerator, PRESETS, type HeaderGeneratorOptions } from 'header-generator';
export { PRESETS } from 'header-generator';
/**
* Checks if a generated user agent is valid (doesn't contain unwanted strings)
*
* @param {string} userAgent The user agent string to validate
* @param {string} browser The browser type (used to determine which filters to apply)
* @returns {boolean} True if the user agent is valid, false if it contains unwanted strings
*/
const isValidUserAgent = (userAgent: string, browser: string): boolean => {
browser = browser.toLowerCase();
if (browser === 'chrome') {
return !(userAgent.includes('Chrome-Lighthouse') || userAgent.includes('Gener8') || userAgent.includes('HeadlessChrome') || userAgent.includes('SMTBot') || userAgent.includes('Electron') || userAgent.includes('Code'));
}
if (browser === 'safari') {
return !userAgent.includes('Applebot');
}
return true;
};
/**
* @param {Partial<HeaderGeneratorOptions>} preset Preset from header-generator package (defaults to PRESETS.MODERN_MACOS_CHROME)
* @returns Headers object with user-agent and additional headers
*/
// Cache for HeaderGenerator instances per preset
const generatorCache = new Map<string, HeaderGenerator>();
export const generateHeaders = (preset: Partial<HeaderGeneratorOptions> = PRESETS.MODERN_MACOS_CHROME) => {
const cacheKey = JSON.stringify(preset);
let generator = generatorCache.get(cacheKey);
if (!generator) {
generator = new HeaderGenerator(preset);
generatorCache.set(cacheKey, generator);
}
let headers = generator.getHeaders();
const userAgent = headers['user-agent'];
let detectedBrowser: string;
if (userAgent.includes('Firefox')) {
detectedBrowser = 'firefox';
} else if (userAgent.includes('Safari') && !userAgent.includes('Chrome')) {
detectedBrowser = 'safari';
} else {
detectedBrowser = 'chrome';
}
let attempts = 0;
while (!isValidUserAgent(headers['user-agent'], detectedBrowser) && attempts < 10) {
headers = generator.getHeaders();
attempts++;
}
return headers;
};
/** List of headers to include from header-generator output
* excluding headers that are typically set manually or by the environment
*/
export const generatedHeaders = new Set([
// 'content-length',
// 'cache-control',
// sec-ch-ua (chrome client hints)
'sec-ch-ua',
'sec-ch-ua-mobile',
'sec-ch-ua-platform',
// 'origin',
// 'content-type',
'upgrade-insecure-requests',
// 'user-agent', // handle manually
'accept',
// sec-fetch (fetch metadata)
'sec-fetch-site',
'sec-fetch-mode',
'sec-fetch-user',
'sec-fetch-dest',
// 'referer', // handle manually
'accept-encoding',
'accept-language',
// 'cookie',
'priority',
]);