Files
RSSHub/lib/utils/request-rewriter/fetch.ts
T
bugyaluwang ff1ff8d940 feat(dev): visualize network requests in the development environment on Chrome Devtools's network tab (#17308)
* chore: add node-network-devtools to enhance development experience

* feat(dev): visualize network requests in the development environment on Chrome Devtools's network tab

* chore(deps): update node-network-devtools

* fix: lock error when ci

* chore: update node-network-devtools to v1.0.18

* chore: update node-network-devtools

* chore: use node-network-devtools v1.0.20

* feat: use hook to add custom headers for fetch

* fix: complex method 16

* test: add test for fetch

* fix: test type error

* fix: hard code

* chore(cr): rename the fetch.spec.ts to fetch.test.ts and use exact version

* chore: fix lock error

---------
2024-11-06 04:50:43 +08:00

83 lines
2.4 KiB
TypeScript

import logger from '@/utils/logger';
import { config } from '@/config';
import undici, { Request, RequestInfo, RequestInit } from 'undici';
import proxy from '@/utils/proxy';
import { RateLimiterMemory, RateLimiterQueue } from 'rate-limiter-flexible';
import { useRegisterRequest } from 'node-network-devtools';
const limiter = new RateLimiterMemory({
points: 10,
duration: 1,
execEvenly: true,
});
const limiterQueue = new RateLimiterQueue(limiter, {
maxQueueSize: 5000,
});
export const useCustomHeader = (headers: Headers) => {
process.env.NODE_ENV === 'dev' &&
useRegisterRequest((req) => {
for (const [key, value] of headers.entries()) {
req.requestHeaders[key] = value;
}
return req;
});
};
const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: RequestInit) => {
const request = new Request(input, init);
const options: RequestInit = {};
logger.debug(`Outgoing request: ${request.method} ${request.url}`);
// ua
if (!request.headers.get('user-agent')) {
request.headers.set('user-agent', config.ua);
}
// accept
if (!request.headers.get('accept')) {
request.headers.set('accept', '*/*');
}
// referer
if (!request.headers.get('referer')) {
try {
const urlHandler = new URL(request.url);
request.headers.set('referer', urlHandler.origin);
} catch {
// ignore
}
}
let isRetry = false;
if (request.headers.get('x-prefer-proxy')) {
isRetry = true;
request.headers.delete('x-prefer-proxy');
}
useCustomHeader(request.headers);
// proxy
if (!init?.dispatcher && proxy.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) {
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
let urlHandler;
try {
urlHandler = new URL(request.url);
} catch {
// ignore
}
if (proxyRegex.test(request.url) && request.url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) {
options.dispatcher = proxy.dispatcher;
logger.debug(`Proxying request: ${request.url}`);
}
}
await limiterQueue.removeTokens(1);
return undici.fetch(request, options);
};
export default wrappedFetch;