* Initial plan * Replace @tonyrl/rand-user-agent with header-generator and add sec-ch-* headers Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * Extract unwanted UA validation into standalone isValidUserAgent function Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * Add headerGeneratorPreset option to got and ofetch utilities * Replace randUserAgent usage in lib/routes with headerGeneratorPreset option Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * chore: pin header-generator to exact match * fix: replace randUserAgent with a static user agent string in config * Remove deprecated _randUserAgent function and related tests Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * refactor: rename to header-generator * Changes before error encountered Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> * fix: update generateHeaders parameter type to Partial<HeaderGeneratorOptions> * fix: add ios preset * fix: rename headerGeneratorPreset to headerGeneratorOptions * feat: add MODERN_IOS preset and update header-generator patch hash * fix: use ios preset * fix: update FetchOptions to include headerGeneratorOptions type * refactor: simplify header generation and update header inclusion list * fix: remove header-generator patch * refactor: update user-agent handling in requests and tests * test: remove redundant headerGeneratorOptions test and add new ofetch header preset test * test: add custom user-agent test for ofetch * test: update user-agent expectation in ofetch headers test * Update lib/utils/ofetch.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor: replace hardcoded headers with generatedHeaders in fetch and get request handlers * refactor: rename test suite from rand-user-agent to header-generator and remove unused dependency * Update lib/utils/header-generator.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: TonyRL <11386903+TonyRL@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
213 lines
7.5 KiB
TypeScript
213 lines
7.5 KiB
TypeScript
import { describe, expect, it, vi } from 'vitest';
|
|
import undici from 'undici';
|
|
import got from 'got';
|
|
import http from 'node:http';
|
|
import { PRESETS } from '@/utils/header-generator';
|
|
|
|
process.env.PROXY_URI = 'http://rsshub.proxy:2333/';
|
|
process.env.PROXY_AUTH = 'rsshubtest';
|
|
process.env.PROXY_URL_REGEX = 'headers';
|
|
|
|
await import('@/utils/request-rewriter');
|
|
const { config } = await import('@/config');
|
|
const { default: ofetch } = await import('@/utils/ofetch');
|
|
|
|
describe('request-rewriter', () => {
|
|
it('fetch', async () => {
|
|
const fetchSpy = vi.spyOn(undici, 'fetch');
|
|
|
|
try {
|
|
await (await fetch('http://rsshub.test/headers')).json();
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
// headers
|
|
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
|
|
expect(headers.get('user-agent')).toBe(config.ua);
|
|
expect(headers.get('accept')).toBeDefined();
|
|
expect(headers.get('referer')).toBe('http://rsshub.test');
|
|
expect(headers.get('sec-ch-ua')).toBeDefined();
|
|
expect(headers.get('sec-ch-ua-mobile')).toBeDefined();
|
|
expect(headers.get('sec-ch-ua-platform')).toBeDefined();
|
|
expect(headers.get('sec-fetch-site')).toBeDefined();
|
|
expect(headers.get('sec-fetch-mode')).toBeDefined();
|
|
expect(headers.get('sec-fetch-user')).toBeDefined();
|
|
expect(headers.get('sec-fetch-dest')).toBeDefined();
|
|
|
|
// proxy
|
|
const options = fetchSpy.mock.lastCall?.[1];
|
|
const agentKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy agent options');
|
|
const agentUri = agentKey ? options?.dispatcher?.[agentKey].uri : null;
|
|
expect(agentUri).toBe(process.env.PROXY_URI);
|
|
|
|
// proxy auth
|
|
const headersKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy headers');
|
|
const agentHeaders = headersKey ? options?.dispatcher?.[headersKey] : null;
|
|
expect(agentHeaders['proxy-authorization']).toBe(`Basic ${process.env.PROXY_AUTH}`);
|
|
|
|
// url regex not match
|
|
{
|
|
try {
|
|
await (await fetch('http://rsshub.test/rss')).json();
|
|
} catch {
|
|
// ignore
|
|
}
|
|
const options = fetchSpy.mock.lastCall?.[1];
|
|
expect(options?.dispatcher).toBeUndefined();
|
|
}
|
|
});
|
|
|
|
it('ofetch', async () => {
|
|
const fetchSpy = vi.spyOn(undici, 'fetch');
|
|
|
|
try {
|
|
await ofetch('http://rsshub.test/headers', {
|
|
retry: 0,
|
|
});
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
// headers
|
|
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
|
|
expect(headers.get('user-agent')).toBe(config.ua);
|
|
expect(headers.get('accept')).toBeDefined();
|
|
expect(headers.get('referer')).toBe('http://rsshub.test');
|
|
expect(headers.get('sec-ch-ua')).toBeDefined();
|
|
expect(headers.get('sec-ch-ua-mobile')).toBeDefined();
|
|
expect(headers.get('sec-ch-ua-platform')).toBeDefined();
|
|
expect(headers.get('sec-fetch-site')).toBeDefined();
|
|
expect(headers.get('sec-fetch-mode')).toBeDefined();
|
|
expect(headers.get('sec-fetch-user')).toBeDefined();
|
|
expect(headers.get('sec-fetch-dest')).toBeDefined();
|
|
|
|
// proxy
|
|
const options = fetchSpy.mock.lastCall?.[1];
|
|
const agentKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy agent options');
|
|
const agentUri = agentKey ? options?.dispatcher?.[agentKey].uri : null;
|
|
expect(agentUri).toBe(process.env.PROXY_URI);
|
|
|
|
// proxy auth
|
|
const headersKey = Object.getOwnPropertySymbols(options?.dispatcher).find((s) => s.description === 'proxy headers');
|
|
const agentHeaders = headersKey ? options?.dispatcher?.[headersKey] : null;
|
|
expect(agentHeaders['proxy-authorization']).toBe(`Basic ${process.env.PROXY_AUTH}`);
|
|
|
|
// url regex not match
|
|
{
|
|
try {
|
|
await ofetch('http://rsshub.test/rss', {
|
|
retry: 0,
|
|
});
|
|
} catch {
|
|
// ignore
|
|
}
|
|
const options = fetchSpy.mock.lastCall?.[1];
|
|
expect(options?.dispatcher).toBeUndefined();
|
|
}
|
|
});
|
|
|
|
it('ofetch custom ua', async () => {
|
|
const fetchSpy = vi.spyOn(undici, 'fetch');
|
|
const userAgent = config.trueUA;
|
|
|
|
try {
|
|
await ofetch('http://rsshub.test/headers', {
|
|
retry: 0,
|
|
headers: {
|
|
'user-agent': userAgent,
|
|
},
|
|
});
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
// headers
|
|
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
|
|
expect(headers.get('user-agent')).toBe(userAgent);
|
|
});
|
|
|
|
it('ofetch header preset', async () => {
|
|
const fetchSpy = vi.spyOn(undici, 'fetch');
|
|
|
|
try {
|
|
await ofetch('http://rsshub.test/headers', {
|
|
retry: 0,
|
|
headerGeneratorOptions: PRESETS.MODERN_WINDOWS_CHROME,
|
|
});
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
// headers
|
|
const headers: Headers = fetchSpy.mock.lastCall?.[0].headers;
|
|
expect(headers.get('user-agent')).toBeDefined();
|
|
expect(headers.get('accept')).toBeDefined();
|
|
expect(headers.get('referer')).toBe('http://rsshub.test');
|
|
expect(headers.get('sec-ch-ua')).toBeDefined();
|
|
expect(headers.get('sec-ch-ua-mobile')).toBe('?0');
|
|
expect(headers.get('sec-ch-ua-platform')).toBe('"Windows"');
|
|
expect(headers.get('sec-fetch-site')).toBeDefined();
|
|
expect(headers.get('sec-fetch-mode')).toBeDefined();
|
|
expect(headers.get('sec-fetch-user')).toBeDefined();
|
|
expect(headers.get('sec-fetch-dest')).toBeDefined();
|
|
});
|
|
|
|
it('http', async () => {
|
|
const httpSpy = vi.spyOn(http, 'request');
|
|
|
|
try {
|
|
await got.get('http://rsshub.test/headers', {
|
|
headers: {
|
|
'user-agent': undefined,
|
|
accept: undefined,
|
|
},
|
|
});
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
// headers
|
|
const options = httpSpy.mock.lastCall?.[1];
|
|
const headers = options?.headers;
|
|
expect(headers?.['user-agent']).toBe(config.ua);
|
|
expect(headers?.accept).toBeDefined();
|
|
expect(headers?.referer).toBe('http://rsshub.test');
|
|
|
|
// proxy
|
|
const agentUri = options?.agent?.proxy?.href;
|
|
expect(agentUri).toBe(process.env.PROXY_URI);
|
|
expect(options?.agent?.proxyHeaders['proxy-authorization']).toBe(`Basic ${process.env.PROXY_AUTH}`);
|
|
|
|
// url regex not match
|
|
{
|
|
try {
|
|
await got.get('http://rsshub.test/rss', {
|
|
headers: {
|
|
'user-agent': undefined,
|
|
accept: undefined,
|
|
},
|
|
});
|
|
} catch {
|
|
// ignore
|
|
}
|
|
const options = httpSpy.mock.lastCall?.[1];
|
|
expect(options?.agent).toBeUndefined();
|
|
}
|
|
});
|
|
|
|
it('rate limiter', async () => {
|
|
const time = Date.now();
|
|
await Promise.all(
|
|
Array.from({ length: 20 }).map(async () => {
|
|
try {
|
|
await fetch('http://rsshub.test/headers');
|
|
} catch {
|
|
// ignore
|
|
}
|
|
})
|
|
);
|
|
expect(Date.now() - time).toBeGreaterThan(1500);
|
|
});
|
|
});
|