xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'request-wrapper'

This commit is contained in:
DIYgod 2019-08-11 00:37:34 +08:00
parent 49f3f59458 e5d34eda6b
commit 5d8cf35ef5
7 changed files with 284 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/index.js
View File

@ -4,6 +4,7 @@ const config = require('./config');
const Koa = require('koa');
const fs = require('fs');
const logger = require('./utils/logger');
require('./utils/request-wrapper');
const onerror = require('./middleware/onerror');
const header = require('./middleware/header');

53
lib/utils/got.js
View File

@ -1,52 +1,8 @@
const logger = require('./logger');
const config = require('@/config');
const SocksProxyAgent = require('socks-proxy-agent');
const tunnel = require('tunnel');
const got = require('got');
const queryString = require('query-string');
let agent = null;
if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
agent = {};
const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
switch (config.proxy.protocol) {
case 'socks':
agent.http = new SocksProxyAgent(proxyUrl);
agent.https = new SocksProxyAgent(proxyUrl);
break;
case 'http':
process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
agent.http = tunnel.httpOverHttp({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
agent.https = tunnel.httpsOverHttp({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
break;
case 'https':
agent.http = tunnel.httpOverHttps({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
agent.https = tunnel.httpsOverHttps({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
break;
}
}
const custom = got.extend({
retry: {
retries: config.requestRetry,
@ -85,20 +41,11 @@ const custom = got.extend({
options.query = options.query.replace(/([\u4e00-\u9fa5])/g, (str) => encodeURIComponent(str));
options.searchParams = options.query; // for Got v11 after
}
if (agent && new RegExp(config.proxy.url_regex).test(options.href)) {
options.agent = agent[options.protocol.slice(0, -1)];
if (config.proxy.auth) {
options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
}
logger.info(`Proxy for ${options.href}`);
}
},
],
},
headers: {
'user-agent': config.ua,
'x-app': 'RSSHub',
},
});
custom.all = (list) => Promise.all(list);

99
lib/utils/request-wrapper.js Normal file
View File

@ -0,0 +1,99 @@
const config = require('@/config');
const SocksProxyAgent = require('socks-proxy-agent');
const tunnel = require('tunnel');
const logger = require('./logger');
const http = require('http');
const https = require('https');
let agent = null;
if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
agent = {};
const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
switch (config.proxy.protocol) {
case 'socks':
agent.http = new SocksProxyAgent(proxyUrl);
agent.https = new SocksProxyAgent(proxyUrl);
break;
case 'http':
process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
agent.http = tunnel.httpOverHttp({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
agent.https = tunnel.httpsOverHttp({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
break;
case 'https':
agent.http = tunnel.httpOverHttps({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
agent.https = tunnel.httpsOverHttps({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
break;
}
}
const requestWrapper = (url, options) => {
if (agent && new RegExp(config.proxy.url_regex).test(url)) {
let agentResult;
try {
agentResult = agent[(options.protocol || url.match(/(https?:)/)[1]).slice(0, -1)];
} catch (error) {
agentResult = null;
}
options.agent = agentResult;
if (config.proxy.auth) {
if (!options.headers) {
options.headers = {};
}
options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
}
logger.info(`Proxy for ${url}`);
}
let hasUA = false;
for (const header in options.headers) {
if (header.toLowerCase() === 'user-agent') {
hasUA = true;
}
}
if (!hasUA) {
if (!options.headers) {
options.headers = {};
}
options.headers['user-agent'] = config.ua;
}
options.headers['x-app'] = 'RSSHub';
};
const httpWrap = (func) => {
const origin = func;
return function(url, request) {
if (typeof url === 'object') {
const req = url;
requestWrapper(req.url || req.href || `${req.protocol}//${req.hostname}${req.path}`, req);
} else {
requestWrapper(url, request);
}
return origin.apply(this, arguments);
};
};
http.get = httpWrap(http.get);
https.get = httpWrap(https.get);
http.request = httpWrap(http.request);
https.request = httpWrap(https.request);

1
lib/utils/rss-parser.js
View File

@ -7,7 +7,6 @@ const parser = new Parser({
},
headers: {
'User-Agent': config.ua,
'X-APP': 'RSSHub',
},
});

134
test/utils/got.js
View File

@ -1,22 +1,13 @@
let got = require('../../lib/utils/got');
const got = require('../../lib/utils/got');
const config = require('../../lib/config');
const nock = require('nock');
afterEach(() => {
delete process.env.PROXY_PROTOCOL;
delete process.env.PROXY_HOST;
delete process.env.PROXY_PORT;
delete process.env.PROXY_AUTH;
delete process.env.PROXY_URL_REGEX;
});
describe('got', () => {
it('headers', async () => {
nock('http://rsshub.test')
.get('/test')
.reply(function() {
expect(this.req.headers['user-agent']).toBe(config.ua);
expect(this.req.headers['x-app']).toBe('RSSHub');
return [200, ''];
});
@ -81,127 +72,4 @@ describe('got', () => {
},
});
});
it('proxy socks', async () => {
process.env.PROXY_PROTOCOL = 'socks';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
jest.resetModules();
got = require('../../lib/utils/got');
nock('http://rsshub.test')
.get('/proxy')
.reply(() => [200, '']);
await got.get('http://rsshub.test/proxy', {
hooks: {
beforeRequest: [
(options) => {
expect(options.agent.constructor.name).toBe('SocksProxyAgent');
expect(options.agent.options.href).toBe('socks://rsshub.proxy:2333');
},
],
},
});
});
it('proxy http', async () => {
process.env.PROXY_PROTOCOL = 'http';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
jest.resetModules();
got = require('../../lib/utils/got');
nock('http://rsshub.test')
.get('/proxy')
.reply(() => [200, '']);
await got.get('http://rsshub.test/proxy', {
hooks: {
beforeRequest: [
(options) => {
expect(options.agent.constructor.name).toBe('TunnelingAgent');
expect(options.agent.options.proxy.host).toBe('rsshub.proxy');
expect(options.agent.options.proxy.port).toBe(2333);
},
],
},
});
});
it('proxy https', async () => {
process.env.PROXY_PROTOCOL = 'https';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
jest.resetModules();
got = require('../../lib/utils/got');
nock('http://rsshub.test')
.get('/proxy')
.reply(() => [200, '']);
await got.get('http://rsshub.test/proxy', {
hooks: {
beforeRequest: [
(options) => {
expect(options.agent.constructor.name).toBe('TunnelingAgent');
expect(options.agent.options.proxy.host).toBe('rsshub.proxy');
expect(options.agent.options.proxy.port).toBe(2333);
},
],
},
});
});
it('auth', async () => {
process.env.PROXY_AUTH = 'testtest';
process.env.PROXY_PROTOCOL = 'socks';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
jest.resetModules();
got = require('../../lib/utils/got');
nock('http://rsshub.test')
.get('/auth')
.reply(function() {
expect(this.req.headers['user-agent']).toBe(config.ua);
expect(this.req.headers['proxy-authorization']).toBe('Basic testtest');
return [200, ''];
});
await got.get('http://rsshub.test/auth');
});
it('url_regex', async () => {
process.env.PROXY_URL_REGEX = 'url_regex';
process.env.PROXY_PROTOCOL = 'socks';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
jest.resetModules();
got = require('../../lib/utils/got');
nock('http://rsshub.test')
.get('/url_regex')
.reply(() => [200, '']);
nock('http://rsshub.test')
.get('/proxy')
.reply(() => [200, '']);
await got.get('http://rsshub.test/url_regex', {
hooks: {
beforeRequest: [
(options) => {
expect(options.agent.constructor.name).toBe('SocksProxyAgent');
expect(options.agent.options.href).toBe('socks://rsshub.proxy:2333');
},
],
},
});
await got.get('http://rsshub.test/proxy', {
hooks: {
beforeRequest: [
(options) => {
expect(options.agent).toBe(undefined);
},
],
},
});
});
});

167
test/utils/request-wrapper.js Normal file
View File

@ -0,0 +1,167 @@
const got = require('../../lib/utils/got');
const parser = require('../../lib/utils/rss-parser');
const nock = require('nock');
require('../../lib/utils/request-wrapper');
let check = () => {};
const simpleResponse = '<rss version="2.0"><channel><item></item></channel></rss>';
afterEach(() => {
delete process.env.PROXY_PROTOCOL;
delete process.env.PROXY_HOST;
delete process.env.PROXY_PORT;
delete process.env.PROXY_AUTH;
delete process.env.PROXY_URL_REGEX;
nock.restore();
nock.activate();
check = () => {};
const http = require('http');
const httpWrap = (func) => {
const origin = func;
return function(url, request) {
if (typeof url === 'object') {
check(url);
} else {
check(request);
}
return origin.apply(this, arguments);
};
};
http.get = httpWrap(http.get);
http.request = httpWrap(http.request);
});
describe('got', () => {
it('headers', async () => {
nock('http://rsshub.test')
.get('/test')
.times(2)
.reply(function() {
expect(this.req.headers['x-app']).toBe('RSSHub');
return [200, simpleResponse];
});
await got.get('http://rsshub.test/test');
await parser.parseURL('http://rsshub.test/test');
});
it('proxy socks', async () => {
process.env.PROXY_PROTOCOL = 'socks';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
jest.resetModules();
require('../../lib/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('SocksProxyAgent');
expect(request.agent.options.href).toBe('socks://rsshub.proxy:2333');
};
nock('http://rsshub.test')
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('proxy http', async () => {
process.env.PROXY_PROTOCOL = 'http';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
jest.resetModules();
require('../../lib/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('TunnelingAgent');
expect(request.agent.options.proxy.host).toBe('rsshub.proxy');
expect(request.agent.options.proxy.port).toBe(2333);
};
nock('http://rsshub.test')
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('proxy https', async () => {
process.env.PROXY_PROTOCOL = 'https';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
jest.resetModules();
require('../../lib/utils/request-wrapper');
check = (request) => {
expect(request.agent.constructor.name).toBe('TunnelingAgent');
expect(request.agent.options.proxy.host).toBe('rsshub.proxy');
expect(request.agent.options.proxy.port).toBe(2333);
};
nock('http://rsshub.test')
.get('/proxy')
.times(2)
.reply(200, simpleResponse);
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
it('auth', async () => {
process.env.PROXY_AUTH = 'testtest';
process.env.PROXY_PROTOCOL = 'socks';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
jest.resetModules();
require('../../lib/utils/request-wrapper');
nock('http://rsshub.test')
.get('/auth')
.times(2)
.reply(function() {
expect(this.req.headers['proxy-authorization']).toBe('Basic testtest');
return [200, simpleResponse];
});
await got.get('http://rsshub.test/auth');
await parser.parseURL('http://rsshub.test/auth');
});
it('url_regex', async () => {
process.env.PROXY_URL_REGEX = 'url_regex';
process.env.PROXY_PROTOCOL = 'socks';
process.env.PROXY_HOST = 'rsshub.proxy';
process.env.PROXY_PORT = '2333';
jest.resetModules();
require('../../lib/utils/request-wrapper');
check = (request) => {
if (request.path === '/url_regex') {
expect(request.agent.constructor.name).toBe('SocksProxyAgent');
expect(request.agent.options.href).toBe('socks://rsshub.proxy:2333');
} else if (request.path === '/proxy') {
expect(request.agent).toBe(undefined);
}
};
nock('http://rsshub.test')
.get('/url_regex')
.times(2)
.reply(() => [200, simpleResponse]);
nock('http://rsshub.test')
.get('/proxy')
.times(2)
.reply(() => [200, simpleResponse]);
await got.get('http://rsshub.test/url_regex');
await parser.parseURL('http://rsshub.test/url_regex');
await got.get('http://rsshub.test/proxy');
await parser.parseURL('http://rsshub.test/proxy');
});
});

16
test/utils/rss-parser.js Normal file
View File

@ -0,0 +1,16 @@
const parser = require('../../lib/utils/rss-parser');
const config = require('../../lib/config');
const nock = require('nock');
describe('got', () => {
it('headers', async () => {
nock('http://rsshub.test')
.get('/test')
.reply(function() {
expect(this.req.headers['user-agent']).toBe(config.ua);
return [200, '<rss version="2.0"><channel><item></item></channel></rss>'];
});
await parser.parseURL('http://rsshub.test/test');
});
});