From 58e930b4cc79c4b6debaafcc688c096dc13af7af Mon Sep 17 00:00:00 2001 From: zengxs Date: Wed, 6 May 2020 22:43:09 +0800 Subject: [PATCH 1/2] support proxy-uri --- lib/config.js | 1 + lib/utils/request-wrapper.js | 17 ++++++++++++++++- package.json | 1 + yarn.lock | 16 ++++++++-------- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/lib/config.js b/lib/config.js index bd3d65277..63851be85 100644 --- a/lib/config.js +++ b/lib/config.js @@ -85,6 +85,7 @@ const calculateValue = () => { }, puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT, loggerLevel: envs.LOGGER_LEVEL || 'info', + proxyUri: envs.PROXY_URI, proxy: { protocol: envs.PROXY_PROTOCOL, host: envs.PROXY_HOST, diff --git a/lib/utils/request-wrapper.js b/lib/utils/request-wrapper.js index 6bbfac996..227568d92 100644 --- a/lib/utils/request-wrapper.js +++ b/lib/utils/request-wrapper.js @@ -1,4 +1,5 @@ const config = require('@/config').value; +const HttpsProxyAgent = require('https-proxy-agent'); const SocksProxyAgent = require('socks-proxy-agent'); const tunnel = require('tunnel'); const logger = require('./logger'); @@ -6,7 +7,21 @@ const http = require('http'); const https = require('https'); let agent = null; -if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) { +if (config.proxyUri && typeof config.proxyUri === 'string') { + let proxy = null; + if (config.proxyUri.startsWith('http')) { + proxy = new HttpsProxyAgent(config.proxyUri); + } else if (config.proxyUri.startsWith('socks')) { + proxy = new SocksProxyAgent(config.proxyUri); + } else { + throw 'Unknown proxy-uri format'; + } + + agent = { + http: proxy, + https: proxy, + }; +} else if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) { agent = {}; const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`; diff --git a/package.json b/package.json index a3eb6d11e..54dae569d 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "googleapis": "49.0.0", "got": "11.0.3", "he": "1.2.0", + "https-proxy-agent": "5.0.0", "iconv-lite": "0.5.1", "jsdom": "16.2.2", "json-bigint": "0.3.0", diff --git a/yarn.lock b/yarn.lock index 9430dc799..faa18cd3c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6175,6 +6175,14 @@ https-browserify@^1.0.0: resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" integrity sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM= +https-proxy-agent@5.0.0, https-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" + integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== + dependencies: + agent-base "6" + debug "4" + https-proxy-agent@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz#702b71fb5520a132a66de1f67541d9e62154d82b" @@ -6183,14 +6191,6 @@ https-proxy-agent@^4.0.0: agent-base "5" debug "4" -https-proxy-agent@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" - integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== - dependencies: - agent-base "6" - debug "4" - human-signals@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" From 910cbbc1700d04f245d86192dcbb2b651f760d2e Mon Sep 17 00:00:00 2001 From: zengxs Date: Wed, 6 May 2020 23:12:43 +0800 Subject: [PATCH 2/2] update docs & test cases --- docs/install/README.md | 24 +++++++++++++- test/utils/request-wrapper.js | 62 +++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/docs/install/README.md b/docs/install/README.md index 2b03316c5..22b47687d 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -303,7 +303,29 @@ RSSHub 支持 `memory` 和 `redis` 两种缓存方式 ### 代理配置 -部分路由反爬严格，可以配置使用代理抓取 +部分路由反爬严格，可以配置使用代理抓取。 + +可通过**代理 URI**或**代理选项**两种方式来配置代理，当两种配置方式同时被设置时，RSSHub 将会使用**代理 URI**中的配置。 + +#### 代理 URI + +`PROXY_URI`: 代理 URI，支持 socks4, socks5, http, https + +> 代理 URI 的格式为: +> +> - `{protocol}://{host}:{port}` +> - `{protocol}://{username}:{password}@{host}:{port}` (带身份凭证) +> +> 一些示例: +> +> - `socks4://127.0.0.1:1080` +> - `socks5://user:pass@127.0.0.1:1080` (用户名为 `user`, 密码为 `pass`) +> - `socks://127.0.0.1:1080` (protocol 为 socks 时表示 `socks5`) +> - `http://127.0.0.1:8080` +> - `http://user:pass@127.0.0.1:8080` +> - `https://127.0.0.1:8443` + +#### 代理选项 `PROXY_PROTOCOL`: 使用代理，支持 socks，http，https diff --git a/test/utils/request-wrapper.js b/test/utils/request-wrapper.js index 165985d38..912b1a855 100644 --- a/test/utils/request-wrapper.js +++ b/test/utils/request-wrapper.js @@ -6,6 +6,7 @@ let check = () => {}; const simpleResponse = ''; afterEach(() => { + delete process.env.PROXY_URI; delete process.env.PROXY_PROTOCOL; delete process.env.PROXY_HOST; delete process.env.PROXY_PORT; @@ -54,6 +55,67 @@ describe('got', () => { await parser.parseURL('http://api.rsshub.test/test'); }); + it('proxy-uri socks', async () => { + process.env.PROXY_URI = 'socks5://user:pass@rsshub.proxy:2333'; + + jest.resetModules(); + require('../../lib/utils/request-wrapper'); + check = (request) => { + expect(request.agent.constructor.name).toBe('SocksProxyAgent'); + expect(request.agent.proxy.host).toBe('rsshub.proxy'); + expect(request.agent.proxy.port).toBe(2333); + }; + + nock(/rsshub\.test/) + .get('/proxy') + .times(2) + .reply(200, simpleResponse); + + await got.get('http://rsshub.test/proxy'); + await parser.parseURL('http://rsshub.test/proxy'); + }); + + it('proxy-uri http', async () => { + process.env.PROXY_URI = 'http://user:pass@rsshub.proxy:2333'; + + jest.resetModules(); + require('../../lib/utils/request-wrapper'); + check = (request) => { + expect(request.agent.constructor.name).toBe('HttpsProxyAgent'); + expect(request.agent.proxy.auth).toBe('user:pass'); + expect(request.agent.proxy.host).toBe('rsshub.proxy'); + expect(request.agent.proxy.port).toBe(2333); + }; + + nock(/rsshub\.test/) + .get('/proxy') + .times(2) + .reply(200, simpleResponse); + + await got.get('http://rsshub.test/proxy'); + await parser.parseURL('http://rsshub.test/proxy'); + }); + + it('proxy-uri https', async () => { + process.env.PROXY_URI = 'https://rsshub.proxy:2333'; + + jest.resetModules(); + require('../../lib/utils/request-wrapper'); + check = (request) => { + expect(request.agent.constructor.name).toBe('HttpsProxyAgent'); + expect(request.agent.proxy.host).toBe('rsshub.proxy'); + expect(request.agent.proxy.port).toBe(2333); + }; + + nock(/rsshub\.test/) + .get('/proxy') + .times(2) + .reply(200, simpleResponse); + + await got.get('http://rsshub.test/proxy'); + await parser.parseURL('http://rsshub.test/proxy'); + }); + it('proxy socks', async () => { process.env.PROXY_PROTOCOL = 'socks'; process.env.PROXY_HOST = 'rsshub.proxy';