Files
RSSHub/lib/utils/request-rewriter.test.ts
T
2ae10a7a02 feat: replace rand-user-agent with header-generator (#20480)
* Initial plan

* Replace @tonyrl/rand-user-agent with header-generator and add sec-ch-* headers

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* Extract unwanted UA validation into standalone isValidUserAgent function

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* Add headerGeneratorPreset option to got and ofetch utilities

* Replace randUserAgent usage in lib/routes with headerGeneratorPreset option

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* chore: pin header-generator to exact match

* fix: replace randUserAgent with a static user agent string in config

* Remove deprecated _randUserAgent function and related tests

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* refactor: rename to header-generator

* Changes before error encountered

Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>

* fix: update generateHeaders parameter type to Partial<HeaderGeneratorOptions>

* fix: add ios preset

* fix: rename headerGeneratorPreset to headerGeneratorOptions

* feat: add MODERN_IOS preset and update header-generator patch hash

* fix: use ios preset

* fix: update FetchOptions to include headerGeneratorOptions type

* refactor: simplify header generation and update header inclusion list

* fix: remove header-generator patch

* refactor: update user-agent handling in requests and tests

* test: remove redundant headerGeneratorOptions test and add new ofetch header preset test

* test: add custom user-agent test for ofetch

* test: update user-agent expectation in ofetch headers test

* Update lib/utils/ofetch.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* refactor: replace hardcoded headers with generatedHeaders in fetch and get request handlers

* refactor: rename test suite from rand-user-agent to header-generator and remove unused dependency

* Update lib/utils/header-generator.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-11-14 00:49:41 +08:00

213 lines
7.5 KiB
TypeScript

import { describe, expect, it, vi } from 'vitest';
import undici from 'undici';
import got from 'got';
import http from 'node:http';
import { PRESETS } from '@/utils/header-generator';
process.env.PROXY_URI = 'http://rsshub.proxy:2333/';
process.env.PROXY_AUTH = 'rsshubtest';
process.env.PROXY_URL_REGEX = 'headers';
await import('@/utils/request-rewriter');
const { config } = await import('@/config');
const { default: ofetch } = await import('@/utils/ofetch');
describe('request-rewriter', () => {
it('fetch', async () => {
const fetchSpy = vi.spyOn(undici, 'fetch');
try {
await (await fetch('http://rsshub.test/headers')).json();
} catch {
// ignore
}
// headers
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
expect(headers.get('user-agent')).toBe(config.ua);
expect(headers.get('accept')).toBeDefined();
expect(headers.get('referer')).toBe('http://rsshub.test');
expect(headers.get('sec-ch-ua')).toBeDefined();
expect(headers.get('sec-ch-ua-mobile')).toBeDefined();
expect(headers.get('sec-ch-ua-platform')).toBeDefined();
expect(headers.get('sec-fetch-site')).toBeDefined();
expect(headers.get('sec-fetch-mode')).toBeDefined();
expect(headers.get('sec-fetch-user')).toBeDefined();
expect(headers.get('sec-fetch-dest')).toBeDefined();
// proxy
const options = fetchSpy.mock.lastCall?.[1];
const agentKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy agent options');
const agentUri = agentKey ? options?.dispatcher?.[agentKey].uri : null;
expect(agentUri).toBe(process.env.PROXY_URI);
// proxy auth
const headersKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy headers');
const agentHeaders = headersKey ? options?.dispatcher?.[headersKey] : null;
expect(agentHeaders['proxy-authorization']).toBe(`Basic ${process.env.PROXY_AUTH}`);
// url regex not match
{
try {
await (await fetch('http://rsshub.test/rss')).json();
} catch {
// ignore
}
const options = fetchSpy.mock.lastCall?.[1];
expect(options?.dispatcher).toBeUndefined();
}
});
it('ofetch', async () => {
const fetchSpy = vi.spyOn(undici, 'fetch');
try {
await ofetch('http://rsshub.test/headers', {
retry: 0,
});
} catch {
// ignore
}
// headers
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
expect(headers.get('user-agent')).toBe(config.ua);
expect(headers.get('accept')).toBeDefined();
expect(headers.get('referer')).toBe('http://rsshub.test');
expect(headers.get('sec-ch-ua')).toBeDefined();
expect(headers.get('sec-ch-ua-mobile')).toBeDefined();
expect(headers.get('sec-ch-ua-platform')).toBeDefined();
expect(headers.get('sec-fetch-site')).toBeDefined();
expect(headers.get('sec-fetch-mode')).toBeDefined();
expect(headers.get('sec-fetch-user')).toBeDefined();
expect(headers.get('sec-fetch-dest')).toBeDefined();
// proxy
const options = fetchSpy.mock.lastCall?.[1];
const agentKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy agent options');
const agentUri = agentKey ? options?.dispatcher?.[agentKey].uri : null;
expect(agentUri).toBe(process.env.PROXY_URI);
// proxy auth
const headersKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy headers');
const agentHeaders = headersKey ? options?.dispatcher?.[headersKey] : null;
expect(agentHeaders['proxy-authorization']).toBe(`Basic ${process.env.PROXY_AUTH}`);
// url regex not match
{
try {
await ofetch('http://rsshub.test/rss', {
retry: 0,
});
} catch {
// ignore
}
const options = fetchSpy.mock.lastCall?.[1];
expect(options?.dispatcher).toBeUndefined();
}
});
it('ofetch custom ua', async () => {
const fetchSpy = vi.spyOn(undici, 'fetch');
const userAgent = config.trueUA;
try {
await ofetch('http://rsshub.test/headers', {
retry: 0,
headers: {
'user-agent': userAgent,
},
});
} catch {
// ignore
}
// headers
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
expect(headers.get('user-agent')).toBe(userAgent);
});
it('ofetch header preset', async () => {
const fetchSpy = vi.spyOn(undici, 'fetch');
try {
await ofetch('http://rsshub.test/headers', {
retry: 0,
headerGeneratorOptions: PRESETS.MODERN_WINDOWS_CHROME,
});
} catch {
// ignore
}
// headers
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
expect(headers.get('user-agent')).toBeDefined();
expect(headers.get('accept')).toBeDefined();
expect(headers.get('referer')).toBe('http://rsshub.test');
expect(headers.get('sec-ch-ua')).toBeDefined();
expect(headers.get('sec-ch-ua-mobile')).toBe('?0');
expect(headers.get('sec-ch-ua-platform')).toBe('"Windows"');
expect(headers.get('sec-fetch-site')).toBeDefined();
expect(headers.get('sec-fetch-mode')).toBeDefined();
expect(headers.get('sec-fetch-user')).toBeDefined();
expect(headers.get('sec-fetch-dest')).toBeDefined();
});
it('http', async () => {
const httpSpy = vi.spyOn(http, 'request');
try {
await got.get('http://rsshub.test/headers', {
headers: {
'user-agent': undefined,
accept: undefined,
},
});
} catch {
// ignore
}
// headers
const options = httpSpy.mock.lastCall?.[1];
const headers = options?.headers;
expect(headers?.['user-agent']).toBe(config.ua);
expect(headers?.accept).toBeDefined();
expect(headers?.referer).toBe('http://rsshub.test');
// proxy
const agentUri = options?.agent?.proxy?.href;
expect(agentUri).toBe(process.env.PROXY_URI);
expect(options?.agent?.proxyHeaders['proxy-authorization']).toBe(`Basic ${process.env.PROXY_AUTH}`);
// url regex not match
{
try {
await got.get('http://rsshub.test/rss', {
headers: {
'user-agent': undefined,
accept: undefined,
},
});
} catch {
// ignore
}
const options = httpSpy.mock.lastCall?.[1];
expect(options?.agent).toBeUndefined();
}
});
it('rate limiter', async () => {
const time = Date.now();
await Promise.all(
Array.from({ length: 20 }).map(async () => {
try {
await fetch('http://rsshub.test/headers');
} catch {
// ignore
}
})
);
expect(Date.now() - time).toBeGreaterThan(1500);
});
});