Merge branch 'use-got-fix-proxy' of https://github.com/fengkx/RSSHub into fengkx-use-got-fix-proxy

lib/routes/aisixiang/column.js

@@ -10,7 +10,7 @@ module.exports = async (ctx) => {
     const host = `http://www.aisixiang.com/data/search.php?lanmu=${id}`;
 
     const response = await axios.get(host, {
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
 
     response.data = iconv.decode(response.data, 'gbk');

lib/routes/aisixiang/ranking.js

@@ -10,7 +10,7 @@ module.exports = async (ctx) => {
     const host = `http://www.aisixiang.com/toplist/index.php?id=${type}&period=${range}`;
 
     const response = await axios.get(host, {
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
 
     response.data = iconv.decode(response.data, 'gbk');

lib/routes/aisixiang/utils.js

@@ -10,13 +10,13 @@ const ProcessFeed = async (link) => {
         .split('.')[0];
 
     let response = await axios.get(`http://www.aisixiang.com/data/view_json.php?id=${id}`, {
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
 
     const description = JSON.parse(iconv.decode(response.data, 'gbk')).content;
     link = `http://www.aisixiang.com${link}`;
     response = await axios.get(link, {
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
 
     response.data = iconv.decode(response.data, 'gbk');

lib/routes/bilibili/cache.js

@@ -65,7 +65,7 @@ module.exports = {
             const nameResponse = await axios({
                 method: 'get',
                 url: `https://www.bilibili.com/video/av${aid}`,
-                responseType: 'arraybuffer',
+                responseType: 'buffer',
             });
             const responseHtml = iconv.decode(nameResponse.data, 'UTF-8');
             const $ = cheerio.load(responseHtml);

lib/routes/bilibili/partion-ranking.js

@@ -1,5 +1,5 @@
 const axios = require('../../utils/axios');
-const axios_ins = axios.create({
+const axios_ins = axios.extend({
     headers: {
         Referer: 'https://www.bilibili.com/',
     },

lib/routes/cartoonmad/comic.js

@@ -21,7 +21,7 @@ const getChapters = ($) =>
 module.exports = async (ctx) => {
     const { id } = ctx.params;
 
-    const { data } = await axios.get(`https://www.cartoonmad.com/comic/${id}`, { responseType: 'arraybuffer' });
+    const { data } = await axios.get(`https://www.cartoonmad.com/comic/${id}`, { responseType: 'buffer' });
     const content = iconv.decode(new Buffer.from(data), 'big5');
     const $ = cheerio.load(content);
 

lib/routes/dapenti/utils.js

@@ -14,7 +14,7 @@ module.exports = {
             // 喷嚏网编码为GBK，需要转码
             // 转码需要设定返回数据的格式，其可选项是arraybuffer,blob,document,json,text,stream
             // 默认为json
-            responseType: 'arraybuffer',
+            responseType: 'buffer',
         });
         // 转码
         const data = iconv.decode(listRes.data, 'gb2312');
@@ -34,7 +34,7 @@ module.exports = {
                     headers: {
                         Referer: url,
                     },
-                    responseType: 'arraybuffer',
+                    responseType: 'buffer',
                 });
                 const convert_data = iconv.decode(original_data.data, 'gbk');
                 const detail_data = cheerio

lib/routes/dongqiudi/utils.js

@@ -40,7 +40,7 @@ const ProcessHref = (content) => {
 };
 
 const ProcessFeed = async (ctx, link, api) => {
-    const axios_ins = axios.create({
+    const axios_ins = axios.extend({
         headers: {
             'X-Requested-With': 'XMLHttpRequest',
             Referer: link,

lib/routes/dytt/index.js

@@ -21,7 +21,7 @@ async function load(link, ctx) {
 
 module.exports = async (ctx) => {
     const response = await axios.get('http://www.dytt8.net', {
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
     response.data = iconv.decode(response.data, 'gb2312');
 

lib/routes/juejin/trending.js

@@ -1,7 +1,7 @@
 const axios = require('../../utils/axios');
 const querystring = require('querystring');
 
-const axios_ins = axios.create({
+const axios_ins = axios.extend({
     responseType: 'json',
     headers: { 'X-Juejin-Src': 'web' },
 });

lib/routes/ltaaa/_article.js

@@ -12,7 +12,7 @@ module.exports = async function get_article(url) {
         method: 'get',
         url: url,
         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0',
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
     const data = response.data;
     const $ = cheerio.load(iconv.decode(data, 'CP936'));

lib/routes/nga/forum.js

@@ -3,7 +3,7 @@ const qs = require('querystring');
 
 module.exports = async (ctx) => {
     const { fid } = ctx.params;
-    const axiosInstance = axios.create({
+    const axiosInstance = axios.extend({
         baseURL: 'https://ngabbs.com/app_api.php',
         headers: {
             'Content-Type': 'application/x-www-form-urlencoded',

lib/routes/nga/post.js

@@ -10,7 +10,7 @@ module.exports = async (ctx) => {
         const response = await axios({
             method: 'get',
             url: link,
-            responseType: 'arraybuffer',
+            responseType: 'buffer',
             headers: {
                 Cookie: `guestJs=${timestamp};`,
             },

lib/routes/novel/biquge.js

@@ -15,7 +15,7 @@ module.exports = async (ctx) => {
             Referer: `${baseUrl}${id}/`,
         },
         // responseEncoding: 'gbk', //该配置项在 0.18版本中没有打包进去
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
     const responseHtml = iconv.decode(response.data, 'GBK');
     const $ = cheerio.load(responseHtml);

lib/routes/novel/booksky.js

@@ -14,7 +14,7 @@ module.exports = async (ctx) => {
             Host: 'booksky.so',
             Referer: 'http://booksky.so/',
         },
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
     const responseHtml = iconv.decode(response.data, 'GBK');
     const $ = cheerio.load(responseHtml);

lib/routes/novel/uukanshu.js

@@ -11,7 +11,7 @@ module.exports = async (ctx) => {
         headers: {
             Referer: 'https://www.uukanshu.com/b/${uid}',
         },
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
 
     const data = iconv.decode(response.data, 'gb2312');

lib/routes/novel/wenxuemi.js

@@ -9,11 +9,11 @@ module.exports = async (ctx) => {
     const id1 = ctx.params.id1; // 小说id
     const id2 = ctx.params.id2;
     const id = '/' + id1 + '/' + id2;
-    const axios_ins = axios.create({
+    const axios_ins = axios.extend({
         headers: {
             Referer: `${mbaseUrl}${id}/`,
         },
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
     const response = await axios_ins.get(`${mbaseUrl}${id}/`);
     const responseHtml = iconv.decode(response.data, 'GBK');

lib/routes/oilprice/index.js

@@ -8,7 +8,7 @@ module.exports = async (ctx) => {
     const response = await axios({
         method: 'get',
         url: link,
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
     const $ = cheerio.load(iconv.decode(response.data, 'gb2312'));
     const $list = $('.pp')

lib/routes/sexinsex/index.js

@@ -5,11 +5,11 @@ const url = require('url');
 
 const base = 'http://sexinsex.net';
 const section = '/bbs/forumdisplay.php?fid=';
-const axios_ins = axios.create({
+const axios_ins = axios.extend({
     headers: {
         Referer: base,
     },
-    responseType: 'arraybuffer',
+    responseType: 'buffer',
 });
 
 const sourceTimezoneOffset = -8;

lib/routes/t66y/index.js

@@ -5,11 +5,11 @@ const url = require('url');
 
 const base = 'http://www.t66y.com';
 const section = 'thread0806.php?fid=';
-const axios_ins = axios.create({
+const axios_ins = axios.extend({
     headers: {
         Referer: base,
     },
-    responseType: 'arraybuffer',
+    responseType: 'buffer',
 });
 
 function killViidii(originUrl) {

lib/routes/t66y/post.js

@@ -4,11 +4,11 @@ const iconv = require('iconv-lite');
 const url = require('url');
 
 const base = 'http://www.t66y.com';
-const axios_ins = axios.create({
+const axios_ins = axios.extend({
     headers: {
         Referer: base,
     },
-    responseType: 'arraybuffer',
+    responseType: 'buffer',
 });
 
 function killViidii(originUrl) {

lib/routes/telegram/stickerpack.js

@@ -36,7 +36,7 @@ module.exports = async (ctx) => {
                 const fileResponse = await axios({
                     method: 'get',
                     url: `https://api.telegram.org/file/bot${config.telegram.token}/${path}`,
-                    responseType: 'arraybuffer',
+                    responseType: 'buffer',
                 });
 
                 const filePath = `tmp/${item.file_id}.png`;

lib/routes/thepaper/utils.js

@@ -5,7 +5,7 @@ const axios = require('../../utils/axios');
 
 module.exports = {
     ProcessFeed: async (link, ctx) => {
-        const axios_ins = axios.create({
+        const axios_ins = axios.extend({
             headers: {
                 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A356 Safari/604.1',
             },

lib/routes/tingshuitz/guangzhou.js

@@ -5,7 +5,7 @@ const iconv = require('iconv-lite');
 module.exports = async (ctx) => {
     const url = 'http://www.gzwatersupply.com/stop/stopgl.html2';
     const response = await axios({
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
         method: 'get',
         url: url,
     });

lib/routes/tingshuitz/xian.js

@@ -5,7 +5,7 @@ const iconv = require('iconv-lite');
 module.exports = async (ctx) => {
     const url = 'http://www.xazls.com/tsgg/index.htm';
     const response = await axios.get(url, {
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
 
     const data = response.data;

lib/routes/tingshuitz/yangjiang.js

@@ -5,7 +5,7 @@ const iconv = require('iconv-lite');
 module.exports = async (ctx) => {
     const url = 'http://www.yjsswjt.com/zxdt_list.jsp?flbz=7';
     const response = await axios.get(url, {
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
 
     const data = response.data;

lib/routes/universities/dlu/jiaowu/news.js

@@ -19,7 +19,7 @@ module.exports = async (ctx) => {
         method: 'get',
         url: base_url,
         // url:link,
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
         headers: {
             Referer: base_url,
         },

lib/routes/universities/dpu/jiaowu/news.js

@@ -18,7 +18,7 @@ module.exports = async (ctx) => {
     const response = await axios({
         method: 'get',
         url: link,
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
         headers: {
             Referer: base_url,
         },

lib/routes/universities/dpu/wlfw/news.js

@@ -17,7 +17,7 @@ module.exports = async (ctx) => {
     const response = await axios({
         method: 'get',
         url: link,
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
         headers: {
             Referer: base_url,
         },

lib/routes/universities/nchu/jwc.js

@@ -18,7 +18,7 @@ module.exports = async (ctx) => {
         headers: {
             Referer: host,
         },
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
     const responseHtml = iconv.decode(response.data, 'GBK');
     const $ = cheerio.load(responseHtml);

lib/routes/universities/nku/jwc/index.js

@@ -15,7 +15,7 @@ module.exports = async (ctx) => {
     const response = await axios({
         method: 'get',
         url: `${dirname}${index}/list.htm`,
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
         headers: {
             Referer: 'http://jwc.nankai.edu.cn/',
         },

lib/routes/universities/sctu/information-engineer-faculty/context.js

@@ -6,7 +6,7 @@ module.exports = async (ctx) => {
     const response = await axios({
         method: 'get',
         url: `http://it.sctu.edu.cn/data.asp?fa=2&fl=12&zi=0&id=${ctx.params.id}`,
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
         headers: {
             Referer: 'http://it.sctu.edu.cn/',
         },

lib/routes/universities/sctu/information-engineer-faculty/index.js

@@ -6,7 +6,7 @@ module.exports = async (ctx) => {
     const response = await axios({
         method: 'get',
         url: 'http://it.sctu.edu.cn/list.asp?fa=2&fl=12&zi=0',
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
         headers: {
             Referer: 'http://it.sctu.edu.cn/',
         },

lib/routes/universities/sctu/jwc/context.js

@@ -6,7 +6,7 @@ module.exports = async (ctx) => {
     const response = await axios({
         method: 'get',
         url: `http://www.sctu.edu.cn/jwc/content.asp?type=12&fl=${ctx.params.type}&id=${ctx.params.id}`,
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
         headers: {
             Referer: 'http://sctu.edu.cn/',
         },

lib/routes/universities/sctu/jwc/index.js

@@ -7,7 +7,7 @@ module.exports = async (ctx) => {
     const response = await axios({
         method: 'get',
         url: `${dirname}list.asp?type=12&fl=${+ctx.params.type === 14 ? 14 : 13}`,
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
         headers: {
             Referer: 'http://www.sctu.edu.cn',
         },

lib/routes/universities/swust/cs.js

@@ -3,11 +3,11 @@ const cheerio = require('cheerio');
 const host = 'http://www.cs.swust.edu.cn/';
 
 module.exports = async (ctx) => {
-    const axios_ins = axios.create({
+    const axios_ins = axios.extend({
         headers: {
             Referer: host,
         },
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
 
     let type = ctx.params.type || 1;

lib/routes/universities/swust/jwc_news.js

@@ -2,7 +2,7 @@ const axios = require('../../../utils/axios');
 
 // 转码需要设定返回数据的格式，其可选项是arraybuffer,blob,document,json,text,stream
 // 默认为json
-const axios_ins = axios.create({ responseType: 'blob' });
+const axios_ins = axios.extend({ responseType: 'text' });
 
 const api = 'http://www.dean.swust.edu.cn/cms/portal/news.json';
 const host = 'http://www.dean.swust.edu.cn/';

lib/routes/universities/swust/jwc_notice.js

@@ -2,7 +2,7 @@ const axios = require('../../../utils/axios');
 
 // 转码需要设定返回数据的格式，其可选项是arraybuffer,blob,document,json,text,stream
 // 默认为json
-const axios_ins = axios.create({ responseType: 'blob' });
+const axios_ins = axios.extend({ responseType: 'text' });
 
 const api = 'http://www.dean.swust.edu.cn/cms/portal/notice.json';
 const page = 'http://www.dean.swust.edu.cn/notice/page/';

lib/routes/wenku8/chapter.js

@@ -9,7 +9,7 @@ module.exports = async (ctx) => {
     const response = await axios({
         method: 'get',
         url: `https://www.wenku8.net/novel/${index}/${id}/index.htm`,
-        responseType: 'arraybuffer',
+        responseType: 'buffer',
     });
 
     const responseHtml = iconv.decode(response.data, 'gbk');

lib/routes/ximalaya/album.js

@@ -1,6 +1,6 @@
 const axios = require('../../utils/axios');
 const baseUrl = 'http://www.ximalaya.com';
-const axios_ins = axios.create({});
+const axios_ins = axios.extend({});
 
 module.exports = async (ctx) => {
     const id = ctx.params.id; // 专辑id

lib/routes/zaobao/realtime.js

@@ -2,7 +2,7 @@ const axios = require('../../utils/axios');
 const cheerio = require('cheerio');
 const baseUrl = 'https://www.zaobao.com.sg';
 const host = 'https://www.zaobao.com.sg/realtime';
-const axios_ins = axios.create({
+const axios_ins = axios.extend({
     headers: {
         Referer: host,
     },

lib/routes/zaobao/znews.js

@@ -1,7 +1,7 @@
 const axios = require('../../utils/axios');
 const cheerio = require('cheerio');
 const baseUrl = 'https://www.zaobao.com.sg';
-const axios_ins = axios.create({
+const axios_ins = axios.extend({
     headers: {
         Referer: baseUrl,
     },

lib/utils/axios.js

@@ -1,39 +1,67 @@
 const logger = require('./logger');
 const config = require('../config');
-const SocksProxyAgent = require('socks-proxy-agent');
-const axiosRetry = require('axios-retry');
+const HttpAgent = require('http-proxy-agent');
+const HttpsAgent = require('https-proxy-agent');
+const SocksAgent = require('socks-proxy-agent');
 
-let axios = require('axios');
-if (config.proxy && config.proxy.protocol && typeof config.proxy.protocol === 'string' && config.proxy.protocol.slice(0, 5) === 'socks' && config.proxy.host && config.proxy.port) {
-    // axios closure lead to recursive invokation on create
-    const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
-    const axiosCpy = axios;
-    // When used directly
-    const dump = axios.create({
-        httpAgent: new SocksProxyAgent(proxyUrl),
-        httpsAgent: new SocksProxyAgent(proxyUrl),
-    });
-    dump.create = function(option, ...args) {
-        option = option || {};
-        option = {
-            httpAgent: new SocksProxyAgent(proxyUrl),
-            httpsAgent: new SocksProxyAgent(proxyUrl),
-            ...option,
-        };
-        return axiosCpy.create(option, ...args);
-    };
-    axios = dump;
+const got = require('got');
+let proxyUrl, agent;
+if (config.proxy && config.proxy.protocol && typeof config.proxy.protocol === 'string' && config.proxy.host && config.proxy.port) {
+    proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
+    switch (config.proxy.protocol.slice(0, 5)) {
+        case 'socks':
+            agent = {
+                http: new SocksAgent(proxyUrl),
+                https: new SocksAgent(proxyUrl),
+            };
+            break;
+        case 'http':
+        case 'https':
+            agent = {
+                http: new HttpAgent(proxyUrl),
+                https: new HttpsAgent(proxyUrl),
+            };
+    }
 }
-axiosRetry(axios, {
-    retries: config.requestRetry,
-    retryCondition: () => true,
-    retryDelay: (count, err) => {
-        logger.error(`Request ${err.config.url} fail, retry attempt #${count}: ${err}`);
-        return 100;
+
+const custom = got.extend({
+    retry: {
+        retries: config.requestRetry,
+        statusCodes: [408, 413, 429, 500, 502, 503, 504, 404], // add 404 to default for unit test
+    },
+    hooks: {
+        beforeRetry: [
+            (options, err, count) => {
+                logger.error(`Request ${err.url} fail, retry attempt #${count}: ${err}`);
+            },
+        ],
+        afterResponse: [
+            (response) => {
+                if (response.headers['content-type'].indexOf('application/json') !== -1) {
+                    try {
+                        response.data = JSON.parse(response.body);
+                    } catch (e) {
+                        response.data = response.body;
+                    }
+                } else {
+                    response.data = response.body;
+                }
+                response.status = response.statusCode;
+                return response;
+            },
+        ],
+        onError: [
+            (error) => {
+                error.response.status = error.response.statusCode;
+                return error;
+            },
+        ],
+    },
+    agent: agent,
+    headers: {
+        'user-agent': config.ua,
+        'x-app': 'RSSHub',
     },
 });
 
-axios.defaults.headers.common['User-Agent'] = config.ua;
-axios.defaults.headers.common['X-APP'] = 'RSSHub';
-
-module.exports = axios;
+module.exports = custom;

package.json

@@ -5,7 +5,7 @@
   "main": "lib/index.js",
   "scripts": {
     "start": "node lib/index.js",
-    "dev": "nodemon lib/index.js",
+    "dev": "nodemon $NODE_DEBUG_OPTION lib/index.js",
     "docs:dev": "vuepress dev docs",
     "docs:build": "vuepress build docs",
     "format": "eslint \"**/*.js\" --fix && prettier \"**/*.{js,json,md}\" --write",
@@ -58,7 +58,10 @@
     "form-data": "^2.3.2",
     "git-rev-sync": "1.12.0",
     "googleapis": "39.2.0",
+    "got": "^9.6.0",
     "he": "^1.1.1",
+    "http-proxy-agent": "^2.1.0",
+    "https-proxy-agent": "^2.2.1",
     "iconv-lite": "0.4.24",
     "imgur": "^0.3.1",
     "json-bigint": "0.3.0",
@@ -71,6 +74,7 @@
     "luxon": "^1.7.1",
     "lz-string": "^1.4.4",
     "markdown-it": "^8.4.2",
+    "nock": "^10.0.6",
     "node-fetch": "^2.3.0",
     "path-to-regexp": "3.0.0",
     "pidusage": "^2.0.17",

test/utils/axios.js

@@ -1,22 +1,22 @@
 const axios = require('../../lib/utils/axios');
-const MockAdapter = require('axios-mock-adapter');
-const mock = new MockAdapter(axios);
 const config = require('../../lib/config');
-
+const nock = require('nock');
 describe('axios', () => {
     it('axios headers', async () => {
-        mock.onGet('/test').reply((axiosConfig) => {
-            expect(axiosConfig.headers['User-Agent']).toBe(config.ua);
-            expect(axiosConfig.headers['X-APP']).toBe('RSSHub');
-            return [
-                200,
-                {
-                    code: 0,
-                },
-            ];
-        });
+        nock('http://fortest.com')
+            .get('/test')
+            .reply(function() {
+                expect(this.req.headers['user-agent']).toBe(config.ua);
+                expect(this.req.headers['x-app']).toBe('RSSHub');
+                return [
+                    200,
+                    {
+                        code: 0,
+                    },
+                ];
+            });
 
-        const response = await axios.get('/test');
+        const response = await axios.get('http://fortest.com/test');
         expect(response.status).toBe(200);
         expect(response.data.code).toBe(0);
     });
@@ -24,34 +24,32 @@ describe('axios', () => {
     it('axios retry', async () => {
         const requestRun = jest.fn();
         let requestTime;
-
-        mock.onGet('/test').reply(() => {
-            requestRun();
-
-            // retryDelay
-            const now = new Date();
-            if (requestTime) {
-                expect(now - requestTime).toBeGreaterThanOrEqual(100);
-                expect(now - requestTime).toBeLessThan(120);
-            }
-            requestTime = new Date();
-
-            return [
-                404,
-                {
-                    code: 1,
-                },
-            ];
-        });
+        nock('http://fortest.com')
+            .get('/testRerty')
+            .times(config.requestRetry + 1)
+            .reply(function() {
+                requestRun();
+                const now = new Date();
+                if (requestTime) {
+                    expect(now - requestTime).toBeGreaterThanOrEqual(100);
+                    expect(now - requestTime).toBeLessThan(120);
+                }
+                requestTime = new Date();
+                return [
+                    404,
+                    {
+                        code: 1,
+                    },
+                ];
+            });
 
         try {
-            await axios.get('/test');
+            await axios.get('http://fortest.com/testRerty');
         } catch (error) {
-            expect(error.response.status).toBe(404);
-            expect(error.response.data.code).toBe(1);
+            expect(error.name).toBe('RequestError');
         }
 
         // retries
-        expect(requestRun).toHaveBeenCalledTimes(config.requestRetry + 1);
+        expect(requestRun).toHaveBeenCalledTimes(config.requestRetry);
     });
 });