diff --git a/lib/utils/request-wrapper.js b/lib/utils/request-wrapper.js index 7af7fdbfc..4641f29e0 100644 --- a/lib/utils/request-wrapper.js +++ b/lib/utils/request-wrapper.js @@ -48,6 +48,7 @@ if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.p } const requestWrapper = (url, options) => { + // agent if (agent && new RegExp(config.proxy.url_regex).test(url)) { let agentResult; try { @@ -65,6 +66,8 @@ const requestWrapper = (url, options) => { } logger.info(`Proxy for ${url}`); } + + // ua let hasUA = false; for (const header in options.headers) { if (header.toLowerCase() === 'user-agent') { @@ -77,7 +80,21 @@ const requestWrapper = (url, options) => { } options.headers['user-agent'] = config.ua; } + + // server options.headers.server = 'RSSHub'; + + // referer + if (!options.headers.referer && !options.headers.Referer) { + const protocol = url.match('https?://'); + url = url.replace(/(https?:)?\/\//, ''); + const domain = url + .split('.') + .slice(-2) + .join('.') + .split('/')[0]; + options.headers.referer = (protocol ? protocol[0] : 'https://') + 'www.' + domain; + } }; const httpWrap = (func) => { diff --git a/test/utils/request-wrapper.js b/test/utils/request-wrapper.js index a42e35c93..df181fb90 100644 --- a/test/utils/request-wrapper.js +++ b/test/utils/request-wrapper.js @@ -35,14 +35,26 @@ afterEach(() => { describe('got', () => { it('headers', async () => { nock('http://rsshub.test') - .get('/test') - .times(2) + .get(/.*/) + .times(3) .reply(function() { expect(this.req.headers.server).toBe('RSSHub'); + expect(this.req.headers.referer).toBe('http://www.rsshub.test'); + return [200, simpleResponse]; + }); + nock(/rsshub\.test/) + .get(/.*/) + .times(2) + .reply(function() { + expect(this.req.headers.referer).toBe('https://www.rsshub.test'); return [200, simpleResponse]; }); await got.get('http://rsshub.test/test'); + await got.get('http://rsshub.test'); + await got.get('rsshub.test/test'); + await got.get('api.rsshub.test/test'); + await parser.parseURL('http://rsshub.test/test'); });