feat(core): auto referer

lib/utils/request-wrapper.js

@@ -48,6 +48,7 @@ if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.p
 }
 
 const requestWrapper = (url, options) => {
+    // agent
     if (agent && new RegExp(config.proxy.url_regex).test(url)) {
         let agentResult;
         try {
@@ -65,6 +66,8 @@ const requestWrapper = (url, options) => {
         }
         logger.info(`Proxy for ${url}`);
     }
+
+    // ua
     let hasUA = false;
     for (const header in options.headers) {
         if (header.toLowerCase() === 'user-agent') {
@@ -77,7 +80,21 @@ const requestWrapper = (url, options) => {
         }
         options.headers['user-agent'] = config.ua;
     }
+
+    // server
     options.headers.server = 'RSSHub';
+
+    // referer
+    if (!options.headers.referer && !options.headers.Referer) {
+        const protocol = url.match('https?://');
+        url = url.replace(/(https?:)?\/\//, '');
+        const domain = url
+            .split('.')
+            .slice(-2)
+            .join('.')
+            .split('/')[0];
+        options.headers.referer = (protocol ? protocol[0] : 'https://') + 'www.' + domain;
+    }
 };
 
 const httpWrap = (func) => {

test/utils/request-wrapper.js

@@ -35,14 +35,26 @@ afterEach(() => {
 describe('got', () => {
     it('headers', async () => {
         nock('http://rsshub.test')
-            .get('/test')
-            .times(2)
+            .get(/.*/)
+            .times(3)
             .reply(function() {
                 expect(this.req.headers.server).toBe('RSSHub');
+                expect(this.req.headers.referer).toBe('http://www.rsshub.test');
+                return [200, simpleResponse];
+            });
+        nock(/rsshub\.test/)
+            .get(/.*/)
+            .times(2)
+            .reply(function() {
+                expect(this.req.headers.referer).toBe('https://www.rsshub.test');
                 return [200, simpleResponse];
             });
 
         await got.get('http://rsshub.test/test');
+        await got.get('http://rsshub.test');
+        await got.get('rsshub.test/test');
+        await got.get('api.rsshub.test/test');
+
         await parser.parseURL('http://rsshub.test/test');
     });