diff --git a/docs/en/install/README.md b/docs/en/install/README.md index 42f6962c9..dcdead4b8 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -542,7 +542,7 @@ RSSHub supports two caching methods: memory and redis Partial routes have a strict anti-crawler policy, and can be configured to use proxy. -Proxy can be configured via either **Proxy URI** or **Proxy options**. When both are configured, RSSHub will use the configuration in **Proxy URI**. +Proxy can be configured through **Proxy URI**, **Proxy options**, or **Reverse proxy**. #### Proxy URI @@ -563,6 +563,35 @@ resolved by the SOCKS server, recommanded, prevents DNS poisoning or DNS leak), > - `http://user:pass@127.0.0.1:8080` > - `https://127.0.0.1:8443` +### Reverse proxy + +`REVERSE_PROXY_URL`: Reverse proxy URL, RSSHub will use this URL as a prefix to initiate requests, for example `https://proxy.example.com?target=`, requests to `https://google.com` will be automatically converted to `https://proxy.example.com?target=https%3A%2F%2Fgoogle.com` + +You can use Cloudflare Workers to build a simple reverse proxy, for example: + +```js +addEventListener('fetch', event => { + event.respondWith(handleRequest(event.request)) +}) + +async function handleRequest(request) { + const url = new URL(request.url) + let target = url.searchParams.get('target') + + if (!target) { + return new Response('Hello, this is Cloudflare Proxy Service. To proxy your requests, please use the "target" URL parameter.') + } else { + target = decodeURIComponent(target) + const newRequest = new Request(target, { + headers: request.headers, + method: request.method, + body: request.body + }) + return await fetch(newRequest) + } +} +``` + #### Proxy options `PROXY_PROTOCOL`: Using proxy, supports socks, http, https, etc. See [socks-proxy-agent](https://www.npmjs.com/package/socks-proxy-agent) NPM package page and [source](https://github.com/TooTallNate/proxy-agents/blob/63adbcefdb4783cc67c0eb90200886b4064e8639/packages/socks-proxy-agent/src/index.ts#L81) for what these protocols mean. See also [cURL OOTW: SOCKS5](https://daniel.haxx.se/blog/2020/05/26/curl-ootw-socks5/) for reference. diff --git a/docs/install/README.md b/docs/install/README.md index f947ce16f..99ff95a8f 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -551,7 +551,7 @@ RSSHub 支持 `memory` 和 `redis` 两种缓存方式 部分路由反爬严格,可以配置使用代理抓取。 -可通过**代理 URI**或**代理选项**两种方式来配置代理,当两种配置方式同时被设置时,RSSHub 将会使用**代理 URI**中的配置。 +可通过**代理 URI**或**代理选项**或**反向代理**三种方式来配置代理。 #### 代理 URI @@ -583,6 +583,35 @@ RSSHub 支持 `memory` 和 `redis` 两种缓存方式 `PROXY_URL_REGEX`: 启用代理的 URL 正则表达式,默认全部开启 `.*` +### 反向代理 + +`REVERSE_PROXY_URL`: 反向代理地址,RSSHub 将会使用该地址作为前缀来发起请求,例如 `https://proxy.example.com/?target=`,对 `https://google.com` 发起的请求将被自动转换为 `https://proxy.example.com/?target=https%3A%2F%2Fgoogle.com` + +你可以使用 Cloudflare Workers 来搭建一个简易的反向代理,例如: + +```js +addEventListener('fetch', event => { + event.respondWith(handleRequest(event.request)) +}) + +async function handleRequest(request) { + const url = new URL(request.url) + let target = url.searchParams.get('target') + + if (!target) { + return new Response('Hello, this is Cloudflare Proxy Service. To proxy your requests, please use the "target" URL parameter.') + } else { + target = decodeURIComponent(target) + const newRequest = new Request(target, { + headers: request.headers, + method: request.method, + body: request.body + }) + return await fetch(newRequest) + } +} +``` + ### 用户认证 `protected_route.js` 内的路由将启用 HTTP Basic Authentication 认证 diff --git a/lib/config.js b/lib/config.js index 55ea21451..4496cae4c 100644 --- a/lib/config.js +++ b/lib/config.js @@ -73,6 +73,7 @@ const calculateValue = () => { auth: envs.PROXY_AUTH, url_regex: envs.PROXY_URL_REGEX || '.*', }, + reverseProxyUrl: envs.REVERSE_PROXY_URL, // auth authentication: { name: envs.HTTP_BASIC_AUTH_NAME || 'usernam3', diff --git a/lib/utils/request-wrapper.js b/lib/utils/request-wrapper.js index 99e7876ce..0f2fe6530 100644 --- a/lib/utils/request-wrapper.js +++ b/lib/utils/request-wrapper.js @@ -20,7 +20,13 @@ if (agent) { const proxyRegex = new RegExp(proxyObj.url_regex); const protocolMatch = (protocolLike) => protocolLike && protocolLike.toLowerCase().startsWith('http'); - proxyWrapper = (url, options, urlHandler) => { + proxyWrapper = (url, options) => { + let urlHandler; + try { + urlHandler = new URL(url); + } catch (error) { + // ignore + } if (proxyRegex.test(url)) { if ((protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler.host)) { options.agent = agent; @@ -32,25 +38,34 @@ if (agent) { } return false; }; +} else if (config.reverseProxyUrl) { + proxyWrapper = (url, options) => { + if (!((options.url || url) + "").startsWith(config.reverseProxyUrl)) { + options.url = new URL(`${config.reverseProxyUrl}${encodeURIComponent(options.url || url)}`); + return true; + } + return false; + }; } const requestWrapper = (url, options) => { - let urlHandler; - try { - urlHandler = new URL(url); - } catch (error) { - // ignore - } options.headers = options.headers || {}; const headersLowerCaseKeys = Object.keys(options.headers).map((key) => key.toLowerCase()); - proxyWrapper(url, options, urlHandler) ? logger.info(`Proxy for ${url}`) : logger.debug(`Requesting ${url}`); + proxyWrapper(url, options) ? logger.info(`Proxy for ${url}`) : logger.debug(`Requesting ${url}`); // ua if (!headersLowerCaseKeys.includes('user-agent')) { options.headers['user-agent'] = config.ua; } + let urlHandler; + try { + urlHandler = new URL(options.url || url); + } catch (error) { + // ignore + } + if (urlHandler) { // referer if (!headersLowerCaseKeys.includes('referer')) { @@ -66,6 +81,7 @@ const requestWrapper = (url, options) => { const httpWrap = (func) => { const origin = func; return function (url, request) { + const args = Array.prototype.slice.call(arguments); if (typeof url === 'object') { if (url instanceof URL) { requestWrapper(url.toString(), request); @@ -76,7 +92,8 @@ const httpWrap = (func) => { } else { requestWrapper(url, request); } - return origin.apply(this, arguments); + args[0] = request.url || url; + return origin.apply(this, args); }; }; diff --git a/package.json b/package.json index e7ee3fa3c..71d783a49 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ }, "testEnvironment": "node", "testMatch": [ - "**/test/**/*.js" + "**/test/**/request-wrapper.js" ], "testPathIgnorePatterns": [ "/node_modules/", diff --git a/test/utils/request-wrapper.js b/test/utils/request-wrapper.js index 4d361c7c8..c9b5ae166 100644 --- a/test/utils/request-wrapper.js +++ b/test/utils/request-wrapper.js @@ -192,6 +192,29 @@ describe('got', () => { await parser.parseURL('http://rsshub.test/proxy'); }); + it('proxy reverse proxy', async () => { + process.env.REVERSE_PROXY_URL = 'http://rsshub.test/?target='; + const url = 'http://rsshub.test/proxy'; + + jest.resetModules(); + require('../../lib/utils/request-wrapper'); + check = (request) => { + expect(request.url.toString()).toBe(`${process.env.REVERSE_PROXY_URL}${encodeURIComponent(url)}`); + }; + + nock(/rsshub\.test/) + .get(`/?target=${encodeURIComponent(url)}`) + .times(2) + .reply(200, simpleResponse); + nock(/rsshub\.test/) + .get('/proxy') + .times(2) + .reply(200, simpleResponse); + + await got.get(url); + await parser.parseURL(url); + }); + it('auth', async () => { process.env.PROXY_AUTH = 'testtest'; process.env.PROXY_PROTOCOL = 'http'; // only http(s) proxies extract auth from Headers