xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: request wrapper

This commit is contained in:
DIYgod 2024-01-21 23:34:25 +08:00
parent 42b6b1c724
commit 2550b6ecf6
No known key found for this signature in database
6 changed files with 159 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/config.ts
View File

@ -4,7 +4,7 @@ import got from 'got';
let envs = process.env;
type Config = {
export type Config = {
disallowRobot: boolean;
enableCluster?: string;
isPackage: boolean;
@ -41,7 +41,6 @@ type Config = {
url_regex: string;
};
proxyStrategy: string;
reverseProxyUrl?: string;
pacUri?: string;
pacScript?: string;
authentication: {
@ -378,7 +377,6 @@ const calculateValue = () => {
url_regex: envs.PROXY_URL_REGEX || '.*',
},
proxyStrategy: envs.PROXY_STRATEGY || 'all', // all / on_retry
reverseProxyUrl: envs.REVERSE_PROXY_URL,
pacUri: envs.PAC_URI,
pacScript: envs.PAC_SCRIPT,
// auth

2
lib/index.ts
View File

@ -1,3 +1,5 @@
import '@/utils/request-wrapper'
import { serve } from '@hono/node-server'
import { Handler, Hono } from 'hono'

23
lib/utils/pac-proxy.js → lib/utils/pac-proxy.ts
View File

@ -1,9 +1,9 @@
const config = require('@/config').value;
const logger = require('./logger');
import { type Config } from '@/config';
import logger from '@/utils/logger';
const possibleProtocol = ['http', 'https', 'ftp', 'file', 'data'];
const pacProxy = (pacUri, pacScript, proxyObj) => {
const pacProxy = (pacUri: Config['pacUri'], pacScript: Config['pacScript'], proxyObj: Config['proxy']) => {
let pacUrlHandler = null;
// Validate PAC_URI / PAC_SCRIPT
@ -17,26 +17,26 @@ const pacProxy = (pacUri, pacScript, proxyObj) => {
if (pacUri && typeof pacUri === 'string') {
try {
pacUrlHandler = new URL(pacUri);
} catch (error) {
pacUri = null;
} catch (error: any) {
pacUri = undefined;
pacUrlHandler = null;
logger.error(`Parse PAC_URI error: ${error.stack}`);
}
} else {
pacUri = null;
pacUri = undefined;
}
// Check if PAC_URI has the right protocol
if (pacUri && !possibleProtocol.includes(pacUrlHandler?.protocol?.replace(':', ''))) {
if (pacUri && (!pacUrlHandler?.protocol || !possibleProtocol.includes(pacUrlHandler.protocol.replace(':', '')))) {
logger.error(`Unsupported PAC protocol: ${pacUrlHandler?.protocol?.replace(':', '')}, expect one of ${possibleProtocol.join(', ')}`);
pacUri = null;
pacUri = undefined;
pacUrlHandler = null;
}
// Validate proxyObj
if (pacUrlHandler) {
proxyObj.host = pacUrlHandler.hostname;
proxyObj.port = Number.parseInt(pacUrlHandler.port) || undefined;
proxyObj.port = pacUrlHandler.port;
proxyObj.protocol = pacUrlHandler.protocol.replace(':', '');
} else {
proxyObj.protocol = proxyObj.host = proxyObj.port = proxyObj.auth = undefined;
@ -69,7 +69,4 @@ const pacProxy = (pacUri, pacScript, proxyObj) => {
};
};
module.exports = {
pacProxy,
...pacProxy(config.pacUri, config.pacScript, config.proxy),
};
export default pacProxy

131
lib/utils/request-wrapper.js
View File

@ -1,131 +0,0 @@
const config = require('@/config').value;
const proxyIsPAC = config.pacUri || config.pacScript;
const { proxyUri, proxyObj, proxyUrlHandler } = proxyIsPAC ? require('./pac-proxy') : require('./unify-proxy');
const logger = require('./logger');
const http = require('http');
const https = require('https');
let agent = null;
if (proxyIsPAC) {
const { PacProxyAgent } = require('pac-proxy-agent');
agent = new PacProxyAgent(`pac+${proxyUri}`);
} else if (proxyUri) {
if (proxyUri.startsWith('http')) {
const { HttpsProxyAgent } = require('https-proxy-agent');
agent = new HttpsProxyAgent(proxyUri);
} else if (proxyUri.startsWith('socks')) {
const { SocksProxyAgent } = require('socks-proxy-agent');
agent = new SocksProxyAgent(proxyUri);
}
}
let proxyWrapper = () => false;
if (agent) {
const proxyRegex = new RegExp(proxyObj.url_regex);
const protocolMatch = (protocolLike) => protocolLike && protocolLike.toLowerCase().startsWith('http');
proxyWrapper = (url, options) => {
let urlHandler;
try {
urlHandler = new URL(url);
} catch {
// ignore
}
if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler.host)) {
options.agent = agent;
if (proxyObj.auth) {
options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`;
}
return true;
}
return false;
};
} else if (config.reverseProxyUrl) {
proxyWrapper = (url, options) => {
const urlIn = options.url?.toString() || url;
const proxyRegex = new RegExp(proxyObj.url_regex);
if (
proxyRegex.test(urlIn) &&
!urlIn.startsWith(config.reverseProxyUrl) &&
!options.cookieJar &&
!options.headers['user-agent']?.startsWith('PixivIOSApp') &&
!options.headers['x-goog-api-client'] &&
options.headers.accept !== 'application/dns-json'
) {
options.url = new URL(`${config.reverseProxyUrl}${encodeURIComponent(urlIn)}`);
return true;
}
return false;
};
}
const requestWrapper = (
url,
options = {
headers: {},
}
) => {
options.headers = options.headers || {};
const headersLowerCaseKeys = new Set(Object.keys(options.headers).map((key) => key.toLowerCase()));
let prxied = false;
if (config.proxyStrategy === 'all') {
prxied = proxyWrapper(url, options);
} else if (config.proxyStrategy === 'on_retry' && options.retryCount) {
prxied = proxyWrapper(url, options);
}
if (prxied) {
logger.http(`Proxy for ${url}`);
} else {
logger.http(`Requesting ${url}`);
}
// ua
if (!headersLowerCaseKeys.has('user-agent')) {
options.headers['user-agent'] = config.ua;
}
// Accept
if (!headersLowerCaseKeys.has('Accept')) {
options.headers.Accept = '*/*';
}
let urlHandler;
try {
urlHandler = new URL(options.url || url);
} catch {
// ignore
}
if (
urlHandler && // referer
!headersLowerCaseKeys.has('referer')
) {
options.headers.referer = urlHandler.origin;
}
};
const httpWrap = (func) => {
const origin = func;
return function (url, request) {
const args = Array.prototype.slice.call(arguments);
if (typeof url === 'object') {
if (url instanceof URL) {
requestWrapper(url.toString(), request);
} else {
const req = url;
requestWrapper(req.url || req.href || `${req.protocol}//${req.hostname || req.host}${req.path}`, req);
}
} else {
requestWrapper(url, request);
}
args[0] = request?.url || url;
return origin.apply(this, args);
};
};
http.get = httpWrap(http.get);
https.get = httpWrap(https.get);
http.request = httpWrap(http.request);
https.request = httpWrap(https.request);

137
lib/utils/request-wrapper.ts Normal file
View File

@ -0,0 +1,137 @@
import { config } from '@/config';
import logger from '@/utils/logger';
import http, { type RequestOptions } from 'node:http';
import https from 'node:https';
const proxyIsPAC = config.pacUri || config.pacScript;
import pacProxy from './pac-proxy';
import unifyProxy from './unify-proxy';
let proxyUri: string | undefined;
let proxyObj: Record<string, any> | undefined;
let proxyUrlHandler: URL | null = null;
if (proxyIsPAC) {
const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy)
proxyUri = proxy.proxyUri;
proxyObj = proxy.proxyObj;
proxyUrlHandler = proxy.proxyUrlHandler;
} else {
const proxy = unifyProxy(config.proxyUri, config.proxy)
proxyUri = proxy.proxyUri;
proxyObj = proxy.proxyObj;
proxyUrlHandler = proxy.proxyUrlHandler;
}
let agent = null;
if (proxyIsPAC) {
const { PacProxyAgent } = require('pac-proxy-agent');
agent = new PacProxyAgent(`pac+${proxyUri}`);
} else if (proxyUri) {
if (proxyUri.startsWith('http')) {
const { HttpsProxyAgent } = require('https-proxy-agent');
agent = new HttpsProxyAgent(proxyUri);
} else if (proxyUri.startsWith('socks')) {
const { SocksProxyAgent } = require('socks-proxy-agent');
agent = new SocksProxyAgent(proxyUri);
}
}
let proxyWrapper: (url: string, options: RequestOptions & {
headers: Record<string, string>;
}) => boolean = () => false;
if (agent) {
const proxyRegex = new RegExp(proxyObj.url_regex);
const protocolMatch = (protocolLike?: string | null) => protocolLike?.toLowerCase().startsWith('http');
proxyWrapper = (url, options) => {
let urlHandler;
try {
urlHandler = new URL(url);
} catch {
// ignore
}
if (proxyRegex.test(url) && (protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler?.host)) {
options.agent = agent;
if (proxyObj.auth) {
options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`;
}
return true;
}
return false;
};
}
const requestWrapper = (
url: string,
options: http.RequestOptions = {},
) => {
options.headers = options.headers || {};
const optionsWithHeaders = options as http.RequestOptions & {
headers: Record<string, string>;
};
const headersLowerCaseKeys = new Set(Object.keys(optionsWithHeaders.headers).map((key) => key.toLowerCase()));
let prxied = false;
if (config.proxyStrategy === 'all') {
prxied = proxyWrapper(url, optionsWithHeaders);
} else if (config.proxyStrategy === 'on_retry' && (optionsWithHeaders as any).retryCount) {
prxied = proxyWrapper(url, optionsWithHeaders);
}
if (prxied) {
logger.debug(`Proxy for ${url}`);
} else {
logger.debug(`Requesting ${url}`);
}
// ua
if (!headersLowerCaseKeys.has('user-agent')) {
options.headers['user-agent'] = config.ua;
}
// Accept
if (!headersLowerCaseKeys.has('Accept')) {
options.headers.Accept = '*/*';
}
let urlHandler;
try {
urlHandler = new URL(url);
} catch {
// ignore
}
if (
urlHandler && // referer
!headersLowerCaseKeys.has('referer')
) {
options.headers.referer = urlHandler.origin;
}
};
const httpWrap = (func: typeof http.request) => {
const origin = func;
const warpped: typeof http.request = function (...args) {
let url: string;
let options: http.RequestOptions;
if (args[0] instanceof URL || typeof args[0] === 'string') {
url = args[0].toString();
options = args[1] as http.RequestOptions;
} else {
options = args[0] as http.RequestOptions;
url = `${options.protocol}//${options.hostname || options.host}${options.path}`
}
requestWrapper(url, options);
// @ts-ignore
return origin.apply(this, args);
};
return warpped;
};
http.get = httpWrap(http.get);
https.get = httpWrap(https.get);
http.request = httpWrap(http.request);
https.request = httpWrap(https.request);

21
lib/utils/unify-proxy.js → lib/utils/unify-proxy.ts
View File

@ -1,15 +1,15 @@
const config = require('@/config').value;
const logger = require('./logger');
import { type Config } from '@/config';
import logger from '@/utils/logger';
const defaultProtocol = 'http';
const possibleProtocol = ['http', 'https', 'socks', 'socks4', 'socks4a', 'socks5', 'socks5h'];
const unifyProxy = (proxyUri, proxyObj) => {
const unifyProxy = (proxyUri: Config['proxyUri'], proxyObj: Config['proxy']) => {
proxyObj = proxyObj || {};
const [oriProxyUri, oriProxyObj] = [proxyUri, proxyObj];
proxyObj = { ...proxyObj };
let proxyUrlHandler;
let proxyUrlHandler: URL | null = null;
// PROXY_URI
if (proxyUri && typeof proxyUri === 'string') {
@ -19,7 +19,7 @@ const unifyProxy = (proxyUri, proxyObj) => {
}
try {
proxyUrlHandler = new URL(proxyUri);
} catch (error) {
} catch (error: any) {
logger.error(`Parse PROXY_URI error: ${error.stack}`);
}
}
@ -52,7 +52,7 @@ const unifyProxy = (proxyUri, proxyObj) => {
} else {
logger.warn('PROXY_PORT is not set, leaving proxy agent to determine');
}
} catch (error) {
} catch (error: any) {
logger.error(`Parse PROXY_HOST error: ${error.stack}`);
}
} else {
@ -90,7 +90,7 @@ const unifyProxy = (proxyUri, proxyObj) => {
}
proxyObj.protocol = protocol;
proxyObj.host = proxyUrlHandler.hostname;
proxyObj.port = Number.parseInt(proxyUrlHandler.port) || undefined;
proxyObj.port = proxyUrlHandler.port;
// trailing slash will cause puppeteer to throw net::ERR_NO_SUPPORTED_PROXIES, trim it
proxyUri = proxyUrlHandler.href.endsWith('/') ? proxyUrlHandler.href.slice(0, -1) : proxyUrlHandler.href;
isProxyValid = true;
@ -103,14 +103,11 @@ const unifyProxy = (proxyUri, proxyObj) => {
logger.error('Proxy is disabled due to misconfiguration');
}
proxyObj.protocol = proxyObj.host = proxyObj.port = proxyObj.auth = undefined;
proxyUri = null;
proxyUri = undefined;
proxyUrlHandler = null;
}
return { proxyUri, proxyObj, proxyUrlHandler };
};
module.exports = {
unifyProxy,
...unifyProxy(config.proxyUri, config.proxy),
};
export default unifyProxy