diff --git a/lib/routes/aisixiang/column.js b/lib/routes/aisixiang/column.js index c2c1068e4..8067d8576 100644 --- a/lib/routes/aisixiang/column.js +++ b/lib/routes/aisixiang/column.js @@ -10,7 +10,7 @@ module.exports = async (ctx) => { const host = `http://www.aisixiang.com/data/search.php?lanmu=${id}`; const response = await axios.get(host, { - responseType: 'arraybuffer', + responseType: 'buffer', }); response.data = iconv.decode(response.data, 'gbk'); diff --git a/lib/routes/aisixiang/ranking.js b/lib/routes/aisixiang/ranking.js index f6b9fe2db..15286e4d1 100644 --- a/lib/routes/aisixiang/ranking.js +++ b/lib/routes/aisixiang/ranking.js @@ -10,7 +10,7 @@ module.exports = async (ctx) => { const host = `http://www.aisixiang.com/toplist/index.php?id=${type}&period=${range}`; const response = await axios.get(host, { - responseType: 'arraybuffer', + responseType: 'buffer', }); response.data = iconv.decode(response.data, 'gbk'); diff --git a/lib/routes/aisixiang/utils.js b/lib/routes/aisixiang/utils.js index 3c1bb1e4f..fd26691c6 100644 --- a/lib/routes/aisixiang/utils.js +++ b/lib/routes/aisixiang/utils.js @@ -10,13 +10,13 @@ const ProcessFeed = async (link) => { .split('.')[0]; let response = await axios.get(`http://www.aisixiang.com/data/view_json.php?id=${id}`, { - responseType: 'arraybuffer', + responseType: 'buffer', }); const description = JSON.parse(iconv.decode(response.data, 'gbk')).content; link = `http://www.aisixiang.com${link}`; response = await axios.get(link, { - responseType: 'arraybuffer', + responseType: 'buffer', }); response.data = iconv.decode(response.data, 'gbk'); diff --git a/lib/routes/bilibili/cache.js b/lib/routes/bilibili/cache.js index 21c7f1b12..a27bad591 100644 --- a/lib/routes/bilibili/cache.js +++ b/lib/routes/bilibili/cache.js @@ -65,7 +65,7 @@ module.exports = { const nameResponse = await axios({ method: 'get', url: `https://www.bilibili.com/video/av${aid}`, - responseType: 'arraybuffer', + responseType: 'buffer', }); const responseHtml = iconv.decode(nameResponse.data, 'UTF-8'); const $ = cheerio.load(responseHtml); diff --git a/lib/routes/bilibili/partion-ranking.js b/lib/routes/bilibili/partion-ranking.js index 5222d677e..7ac6ec493 100644 --- a/lib/routes/bilibili/partion-ranking.js +++ b/lib/routes/bilibili/partion-ranking.js @@ -1,5 +1,5 @@ const axios = require('../../utils/axios'); -const axios_ins = axios.create({ +const axios_ins = axios.extend({ headers: { Referer: 'https://www.bilibili.com/', }, diff --git a/lib/routes/cartoonmad/comic.js b/lib/routes/cartoonmad/comic.js index 66b6bc010..7cffec136 100644 --- a/lib/routes/cartoonmad/comic.js +++ b/lib/routes/cartoonmad/comic.js @@ -21,7 +21,7 @@ const getChapters = ($) => module.exports = async (ctx) => { const { id } = ctx.params; - const { data } = await axios.get(`https://www.cartoonmad.com/comic/${id}`, { responseType: 'arraybuffer' }); + const { data } = await axios.get(`https://www.cartoonmad.com/comic/${id}`, { responseType: 'buffer' }); const content = iconv.decode(new Buffer.from(data), 'big5'); const $ = cheerio.load(content); diff --git a/lib/routes/dapenti/utils.js b/lib/routes/dapenti/utils.js index b02253ccc..1f086fd28 100644 --- a/lib/routes/dapenti/utils.js +++ b/lib/routes/dapenti/utils.js @@ -14,7 +14,7 @@ module.exports = { // 喷嚏网编码为GBK,需要转码 // 转码需要设定返回数据的格式,其可选项是arraybuffer,blob,document,json,text,stream // 默认为json - responseType: 'arraybuffer', + responseType: 'buffer', }); // 转码 const data = iconv.decode(listRes.data, 'gb2312'); @@ -34,7 +34,7 @@ module.exports = { headers: { Referer: url, }, - responseType: 'arraybuffer', + responseType: 'buffer', }); const convert_data = iconv.decode(original_data.data, 'gbk'); const detail_data = cheerio diff --git a/lib/routes/dongqiudi/utils.js b/lib/routes/dongqiudi/utils.js index 178708134..2d2f3b187 100644 --- a/lib/routes/dongqiudi/utils.js +++ b/lib/routes/dongqiudi/utils.js @@ -40,7 +40,7 @@ const ProcessHref = (content) => { }; const ProcessFeed = async (ctx, link, api) => { - const axios_ins = axios.create({ + const axios_ins = axios.extend({ headers: { 'X-Requested-With': 'XMLHttpRequest', Referer: link, diff --git a/lib/routes/dytt/index.js b/lib/routes/dytt/index.js index 738dfaecc..ba81113ed 100644 --- a/lib/routes/dytt/index.js +++ b/lib/routes/dytt/index.js @@ -21,7 +21,7 @@ async function load(link, ctx) { module.exports = async (ctx) => { const response = await axios.get('http://www.dytt8.net', { - responseType: 'arraybuffer', + responseType: 'buffer', }); response.data = iconv.decode(response.data, 'gb2312'); diff --git a/lib/routes/juejin/trending.js b/lib/routes/juejin/trending.js index c22fe4744..56906f823 100644 --- a/lib/routes/juejin/trending.js +++ b/lib/routes/juejin/trending.js @@ -1,7 +1,7 @@ const axios = require('../../utils/axios'); const querystring = require('querystring'); -const axios_ins = axios.create({ +const axios_ins = axios.extend({ responseType: 'json', headers: { 'X-Juejin-Src': 'web' }, }); diff --git a/lib/routes/ltaaa/_article.js b/lib/routes/ltaaa/_article.js index 7cf5247d6..96efc1780 100644 --- a/lib/routes/ltaaa/_article.js +++ b/lib/routes/ltaaa/_article.js @@ -12,7 +12,7 @@ module.exports = async function get_article(url) { method: 'get', url: url, 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', - responseType: 'arraybuffer', + responseType: 'buffer', }); const data = response.data; const $ = cheerio.load(iconv.decode(data, 'CP936')); diff --git a/lib/routes/nga/forum.js b/lib/routes/nga/forum.js index 3bffd25d2..7c6766735 100644 --- a/lib/routes/nga/forum.js +++ b/lib/routes/nga/forum.js @@ -3,7 +3,7 @@ const qs = require('querystring'); module.exports = async (ctx) => { const { fid } = ctx.params; - const axiosInstance = axios.create({ + const axiosInstance = axios.extend({ baseURL: 'https://ngabbs.com/app_api.php', headers: { 'Content-Type': 'application/x-www-form-urlencoded', diff --git a/lib/routes/nga/post.js b/lib/routes/nga/post.js index 30186ae60..90a4c6331 100644 --- a/lib/routes/nga/post.js +++ b/lib/routes/nga/post.js @@ -10,7 +10,7 @@ module.exports = async (ctx) => { const response = await axios({ method: 'get', url: link, - responseType: 'arraybuffer', + responseType: 'buffer', headers: { Cookie: `guestJs=${timestamp};`, }, diff --git a/lib/routes/novel/biquge.js b/lib/routes/novel/biquge.js index 89d1832f3..f94eb2f59 100644 --- a/lib/routes/novel/biquge.js +++ b/lib/routes/novel/biquge.js @@ -15,7 +15,7 @@ module.exports = async (ctx) => { Referer: `${baseUrl}${id}/`, }, // responseEncoding: 'gbk', //该配置项在 0.18版本中没有打包进去 - responseType: 'arraybuffer', + responseType: 'buffer', }); const responseHtml = iconv.decode(response.data, 'GBK'); const $ = cheerio.load(responseHtml); diff --git a/lib/routes/novel/booksky.js b/lib/routes/novel/booksky.js index c48039216..b9bf2b724 100644 --- a/lib/routes/novel/booksky.js +++ b/lib/routes/novel/booksky.js @@ -14,7 +14,7 @@ module.exports = async (ctx) => { Host: 'booksky.so', Referer: 'http://booksky.so/', }, - responseType: 'arraybuffer', + responseType: 'buffer', }); const responseHtml = iconv.decode(response.data, 'GBK'); const $ = cheerio.load(responseHtml); diff --git a/lib/routes/novel/uukanshu.js b/lib/routes/novel/uukanshu.js index 1189e63a8..0f49403dd 100644 --- a/lib/routes/novel/uukanshu.js +++ b/lib/routes/novel/uukanshu.js @@ -11,7 +11,7 @@ module.exports = async (ctx) => { headers: { Referer: 'https://www.uukanshu.com/b/${uid}', }, - responseType: 'arraybuffer', + responseType: 'buffer', }); const data = iconv.decode(response.data, 'gb2312'); diff --git a/lib/routes/novel/wenxuemi.js b/lib/routes/novel/wenxuemi.js index 5f03f3a2d..64cbf9e75 100644 --- a/lib/routes/novel/wenxuemi.js +++ b/lib/routes/novel/wenxuemi.js @@ -9,11 +9,11 @@ module.exports = async (ctx) => { const id1 = ctx.params.id1; // 小说id const id2 = ctx.params.id2; const id = '/' + id1 + '/' + id2; - const axios_ins = axios.create({ + const axios_ins = axios.extend({ headers: { Referer: `${mbaseUrl}${id}/`, }, - responseType: 'arraybuffer', + responseType: 'buffer', }); const response = await axios_ins.get(`${mbaseUrl}${id}/`); const responseHtml = iconv.decode(response.data, 'GBK'); diff --git a/lib/routes/oilprice/index.js b/lib/routes/oilprice/index.js index c6010a378..4ec5d570c 100644 --- a/lib/routes/oilprice/index.js +++ b/lib/routes/oilprice/index.js @@ -8,7 +8,7 @@ module.exports = async (ctx) => { const response = await axios({ method: 'get', url: link, - responseType: 'arraybuffer', + responseType: 'buffer', }); const $ = cheerio.load(iconv.decode(response.data, 'gb2312')); const $list = $('.pp') diff --git a/lib/routes/sexinsex/index.js b/lib/routes/sexinsex/index.js index b0bec05c1..78dc5036e 100644 --- a/lib/routes/sexinsex/index.js +++ b/lib/routes/sexinsex/index.js @@ -5,11 +5,11 @@ const url = require('url'); const base = 'http://sexinsex.net'; const section = '/bbs/forumdisplay.php?fid='; -const axios_ins = axios.create({ +const axios_ins = axios.extend({ headers: { Referer: base, }, - responseType: 'arraybuffer', + responseType: 'buffer', }); const sourceTimezoneOffset = -8; diff --git a/lib/routes/t66y/index.js b/lib/routes/t66y/index.js index 96cd6898e..217b70097 100644 --- a/lib/routes/t66y/index.js +++ b/lib/routes/t66y/index.js @@ -5,11 +5,11 @@ const url = require('url'); const base = 'http://www.t66y.com'; const section = 'thread0806.php?fid='; -const axios_ins = axios.create({ +const axios_ins = axios.extend({ headers: { Referer: base, }, - responseType: 'arraybuffer', + responseType: 'buffer', }); function killViidii(originUrl) { diff --git a/lib/routes/t66y/post.js b/lib/routes/t66y/post.js index 1e4452b08..73ce9b53f 100644 --- a/lib/routes/t66y/post.js +++ b/lib/routes/t66y/post.js @@ -4,11 +4,11 @@ const iconv = require('iconv-lite'); const url = require('url'); const base = 'http://www.t66y.com'; -const axios_ins = axios.create({ +const axios_ins = axios.extend({ headers: { Referer: base, }, - responseType: 'arraybuffer', + responseType: 'buffer', }); function killViidii(originUrl) { diff --git a/lib/routes/telegram/stickerpack.js b/lib/routes/telegram/stickerpack.js index 7be28fa6f..6d1423519 100644 --- a/lib/routes/telegram/stickerpack.js +++ b/lib/routes/telegram/stickerpack.js @@ -36,7 +36,7 @@ module.exports = async (ctx) => { const fileResponse = await axios({ method: 'get', url: `https://api.telegram.org/file/bot${config.telegram.token}/${path}`, - responseType: 'arraybuffer', + responseType: 'buffer', }); const filePath = `tmp/${item.file_id}.png`; diff --git a/lib/routes/thepaper/utils.js b/lib/routes/thepaper/utils.js index 49fea84cf..0a9677a2b 100644 --- a/lib/routes/thepaper/utils.js +++ b/lib/routes/thepaper/utils.js @@ -5,7 +5,7 @@ const axios = require('../../utils/axios'); module.exports = { ProcessFeed: async (link, ctx) => { - const axios_ins = axios.create({ + const axios_ins = axios.extend({ headers: { 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A356 Safari/604.1', }, diff --git a/lib/routes/tingshuitz/guangzhou.js b/lib/routes/tingshuitz/guangzhou.js index 8b4b94b37..68a3daeda 100644 --- a/lib/routes/tingshuitz/guangzhou.js +++ b/lib/routes/tingshuitz/guangzhou.js @@ -5,7 +5,7 @@ const iconv = require('iconv-lite'); module.exports = async (ctx) => { const url = 'http://www.gzwatersupply.com/stop/stopgl.html2'; const response = await axios({ - responseType: 'arraybuffer', + responseType: 'buffer', method: 'get', url: url, }); diff --git a/lib/routes/tingshuitz/xian.js b/lib/routes/tingshuitz/xian.js index 9caddd69e..6243a0e33 100644 --- a/lib/routes/tingshuitz/xian.js +++ b/lib/routes/tingshuitz/xian.js @@ -5,7 +5,7 @@ const iconv = require('iconv-lite'); module.exports = async (ctx) => { const url = 'http://www.xazls.com/tsgg/index.htm'; const response = await axios.get(url, { - responseType: 'arraybuffer', + responseType: 'buffer', }); const data = response.data; diff --git a/lib/routes/tingshuitz/yangjiang.js b/lib/routes/tingshuitz/yangjiang.js index fbbdeea5a..07d29574d 100644 --- a/lib/routes/tingshuitz/yangjiang.js +++ b/lib/routes/tingshuitz/yangjiang.js @@ -5,7 +5,7 @@ const iconv = require('iconv-lite'); module.exports = async (ctx) => { const url = 'http://www.yjsswjt.com/zxdt_list.jsp?flbz=7'; const response = await axios.get(url, { - responseType: 'arraybuffer', + responseType: 'buffer', }); const data = response.data; diff --git a/lib/routes/universities/dlu/jiaowu/news.js b/lib/routes/universities/dlu/jiaowu/news.js index c7c242935..86149f282 100644 --- a/lib/routes/universities/dlu/jiaowu/news.js +++ b/lib/routes/universities/dlu/jiaowu/news.js @@ -19,7 +19,7 @@ module.exports = async (ctx) => { method: 'get', url: base_url, // url:link, - responseType: 'arraybuffer', + responseType: 'buffer', headers: { Referer: base_url, }, diff --git a/lib/routes/universities/dpu/jiaowu/news.js b/lib/routes/universities/dpu/jiaowu/news.js index 71adb8118..5e740226c 100644 --- a/lib/routes/universities/dpu/jiaowu/news.js +++ b/lib/routes/universities/dpu/jiaowu/news.js @@ -18,7 +18,7 @@ module.exports = async (ctx) => { const response = await axios({ method: 'get', url: link, - responseType: 'arraybuffer', + responseType: 'buffer', headers: { Referer: base_url, }, diff --git a/lib/routes/universities/dpu/wlfw/news.js b/lib/routes/universities/dpu/wlfw/news.js index 2abeb6bde..5e45a48a3 100644 --- a/lib/routes/universities/dpu/wlfw/news.js +++ b/lib/routes/universities/dpu/wlfw/news.js @@ -17,7 +17,7 @@ module.exports = async (ctx) => { const response = await axios({ method: 'get', url: link, - responseType: 'arraybuffer', + responseType: 'buffer', headers: { Referer: base_url, }, diff --git a/lib/routes/universities/nchu/jwc.js b/lib/routes/universities/nchu/jwc.js index 1e7b46d7d..6d391849f 100644 --- a/lib/routes/universities/nchu/jwc.js +++ b/lib/routes/universities/nchu/jwc.js @@ -18,7 +18,7 @@ module.exports = async (ctx) => { headers: { Referer: host, }, - responseType: 'arraybuffer', + responseType: 'buffer', }); const responseHtml = iconv.decode(response.data, 'GBK'); const $ = cheerio.load(responseHtml); diff --git a/lib/routes/universities/nku/jwc/index.js b/lib/routes/universities/nku/jwc/index.js index b07c4238f..b8a5fb634 100644 --- a/lib/routes/universities/nku/jwc/index.js +++ b/lib/routes/universities/nku/jwc/index.js @@ -15,7 +15,7 @@ module.exports = async (ctx) => { const response = await axios({ method: 'get', url: `${dirname}${index}/list.htm`, - responseType: 'arraybuffer', + responseType: 'buffer', headers: { Referer: 'http://jwc.nankai.edu.cn/', }, diff --git a/lib/routes/universities/sctu/information-engineer-faculty/context.js b/lib/routes/universities/sctu/information-engineer-faculty/context.js index 119fd5f58..998ef3cd5 100644 --- a/lib/routes/universities/sctu/information-engineer-faculty/context.js +++ b/lib/routes/universities/sctu/information-engineer-faculty/context.js @@ -6,7 +6,7 @@ module.exports = async (ctx) => { const response = await axios({ method: 'get', url: `http://it.sctu.edu.cn/data.asp?fa=2&fl=12&zi=0&id=${ctx.params.id}`, - responseType: 'arraybuffer', + responseType: 'buffer', headers: { Referer: 'http://it.sctu.edu.cn/', }, diff --git a/lib/routes/universities/sctu/information-engineer-faculty/index.js b/lib/routes/universities/sctu/information-engineer-faculty/index.js index d526db16b..9d9053b08 100644 --- a/lib/routes/universities/sctu/information-engineer-faculty/index.js +++ b/lib/routes/universities/sctu/information-engineer-faculty/index.js @@ -6,7 +6,7 @@ module.exports = async (ctx) => { const response = await axios({ method: 'get', url: 'http://it.sctu.edu.cn/list.asp?fa=2&fl=12&zi=0', - responseType: 'arraybuffer', + responseType: 'buffer', headers: { Referer: 'http://it.sctu.edu.cn/', }, diff --git a/lib/routes/universities/sctu/jwc/context.js b/lib/routes/universities/sctu/jwc/context.js index e7c1d0885..d48917cd4 100644 --- a/lib/routes/universities/sctu/jwc/context.js +++ b/lib/routes/universities/sctu/jwc/context.js @@ -6,7 +6,7 @@ module.exports = async (ctx) => { const response = await axios({ method: 'get', url: `http://www.sctu.edu.cn/jwc/content.asp?type=12&fl=${ctx.params.type}&id=${ctx.params.id}`, - responseType: 'arraybuffer', + responseType: 'buffer', headers: { Referer: 'http://sctu.edu.cn/', }, diff --git a/lib/routes/universities/sctu/jwc/index.js b/lib/routes/universities/sctu/jwc/index.js index 60df64d51..2cd17017e 100644 --- a/lib/routes/universities/sctu/jwc/index.js +++ b/lib/routes/universities/sctu/jwc/index.js @@ -7,7 +7,7 @@ module.exports = async (ctx) => { const response = await axios({ method: 'get', url: `${dirname}list.asp?type=12&fl=${+ctx.params.type === 14 ? 14 : 13}`, - responseType: 'arraybuffer', + responseType: 'buffer', headers: { Referer: 'http://www.sctu.edu.cn', }, diff --git a/lib/routes/universities/swust/cs.js b/lib/routes/universities/swust/cs.js index 11aa3e37a..f3b5157c4 100644 --- a/lib/routes/universities/swust/cs.js +++ b/lib/routes/universities/swust/cs.js @@ -3,11 +3,11 @@ const cheerio = require('cheerio'); const host = 'http://www.cs.swust.edu.cn/'; module.exports = async (ctx) => { - const axios_ins = axios.create({ + const axios_ins = axios.extend({ headers: { Referer: host, }, - responseType: 'arraybuffer', + responseType: 'buffer', }); let type = ctx.params.type || 1; diff --git a/lib/routes/universities/swust/jwc_news.js b/lib/routes/universities/swust/jwc_news.js index d84ca3179..87fb6471d 100644 --- a/lib/routes/universities/swust/jwc_news.js +++ b/lib/routes/universities/swust/jwc_news.js @@ -2,7 +2,7 @@ const axios = require('../../../utils/axios'); // 转码需要设定返回数据的格式,其可选项是arraybuffer,blob,document,json,text,stream // 默认为json -const axios_ins = axios.create({ responseType: 'blob' }); +const axios_ins = axios.extend({ responseType: 'text' }); const api = 'http://www.dean.swust.edu.cn/cms/portal/news.json'; const host = 'http://www.dean.swust.edu.cn/'; diff --git a/lib/routes/universities/swust/jwc_notice.js b/lib/routes/universities/swust/jwc_notice.js index 1b2ad95b1..f5b488230 100644 --- a/lib/routes/universities/swust/jwc_notice.js +++ b/lib/routes/universities/swust/jwc_notice.js @@ -2,7 +2,7 @@ const axios = require('../../../utils/axios'); // 转码需要设定返回数据的格式,其可选项是arraybuffer,blob,document,json,text,stream // 默认为json -const axios_ins = axios.create({ responseType: 'blob' }); +const axios_ins = axios.extend({ responseType: 'text' }); const api = 'http://www.dean.swust.edu.cn/cms/portal/notice.json'; const page = 'http://www.dean.swust.edu.cn/notice/page/'; diff --git a/lib/routes/wenku8/chapter.js b/lib/routes/wenku8/chapter.js index 20a640d22..00ce4ddcc 100644 --- a/lib/routes/wenku8/chapter.js +++ b/lib/routes/wenku8/chapter.js @@ -9,7 +9,7 @@ module.exports = async (ctx) => { const response = await axios({ method: 'get', url: `https://www.wenku8.net/novel/${index}/${id}/index.htm`, - responseType: 'arraybuffer', + responseType: 'buffer', }); const responseHtml = iconv.decode(response.data, 'gbk'); diff --git a/lib/routes/ximalaya/album.js b/lib/routes/ximalaya/album.js index 1c442f82b..312f4b279 100644 --- a/lib/routes/ximalaya/album.js +++ b/lib/routes/ximalaya/album.js @@ -1,6 +1,6 @@ const axios = require('../../utils/axios'); const baseUrl = 'http://www.ximalaya.com'; -const axios_ins = axios.create({}); +const axios_ins = axios.extend({}); module.exports = async (ctx) => { const id = ctx.params.id; // 专辑id diff --git a/lib/routes/zaobao/realtime.js b/lib/routes/zaobao/realtime.js index b7ca8ca0a..dd46fe8b1 100644 --- a/lib/routes/zaobao/realtime.js +++ b/lib/routes/zaobao/realtime.js @@ -2,7 +2,7 @@ const axios = require('../../utils/axios'); const cheerio = require('cheerio'); const baseUrl = 'https://www.zaobao.com.sg'; const host = 'https://www.zaobao.com.sg/realtime'; -const axios_ins = axios.create({ +const axios_ins = axios.extend({ headers: { Referer: host, }, diff --git a/lib/routes/zaobao/znews.js b/lib/routes/zaobao/znews.js index 95046deb2..cdc5f2f1d 100644 --- a/lib/routes/zaobao/znews.js +++ b/lib/routes/zaobao/znews.js @@ -1,7 +1,7 @@ const axios = require('../../utils/axios'); const cheerio = require('cheerio'); const baseUrl = 'https://www.zaobao.com.sg'; -const axios_ins = axios.create({ +const axios_ins = axios.extend({ headers: { Referer: baseUrl, }, diff --git a/lib/utils/axios.js b/lib/utils/axios.js index b65a0142d..c77c4d481 100644 --- a/lib/utils/axios.js +++ b/lib/utils/axios.js @@ -1,39 +1,67 @@ const logger = require('./logger'); const config = require('../config'); -const SocksProxyAgent = require('socks-proxy-agent'); -const axiosRetry = require('axios-retry'); +const HttpAgent = require('http-proxy-agent'); +const HttpsAgent = require('https-proxy-agent'); +const SocksAgent = require('socks-proxy-agent'); -let axios = require('axios'); -if (config.proxy && config.proxy.protocol && typeof config.proxy.protocol === 'string' && config.proxy.protocol.slice(0, 5) === 'socks' && config.proxy.host && config.proxy.port) { - // axios closure lead to recursive invokation on create - const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`; - const axiosCpy = axios; - // When used directly - const dump = axios.create({ - httpAgent: new SocksProxyAgent(proxyUrl), - httpsAgent: new SocksProxyAgent(proxyUrl), - }); - dump.create = function(option, ...args) { - option = option || {}; - option = { - httpAgent: new SocksProxyAgent(proxyUrl), - httpsAgent: new SocksProxyAgent(proxyUrl), - ...option, - }; - return axiosCpy.create(option, ...args); - }; - axios = dump; +const got = require('got'); +let proxyUrl, agent; +if (config.proxy && config.proxy.protocol && typeof config.proxy.protocol === 'string' && config.proxy.host && config.proxy.port) { + proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`; + switch (config.proxy.protocol.slice(0, 5)) { + case 'socks': + agent = { + http: new SocksAgent(proxyUrl), + https: new SocksAgent(proxyUrl), + }; + break; + case 'http': + case 'https': + agent = { + http: new HttpAgent(proxyUrl), + https: new HttpsAgent(proxyUrl), + }; + } } -axiosRetry(axios, { - retries: config.requestRetry, - retryCondition: () => true, - retryDelay: (count, err) => { - logger.error(`Request ${err.config.url} fail, retry attempt #${count}: ${err}`); - return 100; + +const custom = got.extend({ + retry: { + retries: config.requestRetry, + statusCodes: [408, 413, 429, 500, 502, 503, 504, 404], // add 404 to default for unit test + }, + hooks: { + beforeRetry: [ + (options, err, count) => { + logger.error(`Request ${err.url} fail, retry attempt #${count}: ${err}`); + }, + ], + afterResponse: [ + (response) => { + if (response.headers['content-type'].indexOf('application/json') !== -1) { + try { + response.data = JSON.parse(response.body); + } catch (e) { + response.data = response.body; + } + } else { + response.data = response.body; + } + response.status = response.statusCode; + return response; + }, + ], + onError: [ + (error) => { + error.response.status = error.response.statusCode; + return error; + }, + ], + }, + agent: agent, + headers: { + 'user-agent': config.ua, + 'x-app': 'RSSHub', }, }); -axios.defaults.headers.common['User-Agent'] = config.ua; -axios.defaults.headers.common['X-APP'] = 'RSSHub'; - -module.exports = axios; +module.exports = custom; diff --git a/package.json b/package.json index f38b50451..124ae1bd4 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "scripts": { "start": "node lib/index.js", - "dev": "nodemon lib/index.js", + "dev": "nodemon $NODE_DEBUG_OPTION lib/index.js", "docs:dev": "vuepress dev docs", "docs:build": "vuepress build docs", "format": "eslint \"**/*.js\" --fix && prettier \"**/*.{js,json,md}\" --write", @@ -58,7 +58,10 @@ "form-data": "^2.3.2", "git-rev-sync": "1.12.0", "googleapis": "39.2.0", + "got": "^9.6.0", "he": "^1.1.1", + "http-proxy-agent": "^2.1.0", + "https-proxy-agent": "^2.2.1", "iconv-lite": "0.4.24", "imgur": "^0.3.1", "json-bigint": "0.3.0", @@ -71,6 +74,7 @@ "luxon": "^1.7.1", "lz-string": "^1.4.4", "markdown-it": "^8.4.2", + "nock": "^10.0.6", "node-fetch": "^2.3.0", "path-to-regexp": "3.0.0", "pidusage": "^2.0.17", diff --git a/test/utils/axios.js b/test/utils/axios.js index 1561fb929..1a5430cda 100644 --- a/test/utils/axios.js +++ b/test/utils/axios.js @@ -1,22 +1,22 @@ const axios = require('../../lib/utils/axios'); -const MockAdapter = require('axios-mock-adapter'); -const mock = new MockAdapter(axios); const config = require('../../lib/config'); - +const nock = require('nock'); describe('axios', () => { it('axios headers', async () => { - mock.onGet('/test').reply((axiosConfig) => { - expect(axiosConfig.headers['User-Agent']).toBe(config.ua); - expect(axiosConfig.headers['X-APP']).toBe('RSSHub'); - return [ - 200, - { - code: 0, - }, - ]; - }); + nock('http://fortest.com') + .get('/test') + .reply(function() { + expect(this.req.headers['user-agent']).toBe(config.ua); + expect(this.req.headers['x-app']).toBe('RSSHub'); + return [ + 200, + { + code: 0, + }, + ]; + }); - const response = await axios.get('/test'); + const response = await axios.get('http://fortest.com/test'); expect(response.status).toBe(200); expect(response.data.code).toBe(0); }); @@ -24,34 +24,32 @@ describe('axios', () => { it('axios retry', async () => { const requestRun = jest.fn(); let requestTime; - - mock.onGet('/test').reply(() => { - requestRun(); - - // retryDelay - const now = new Date(); - if (requestTime) { - expect(now - requestTime).toBeGreaterThanOrEqual(100); - expect(now - requestTime).toBeLessThan(120); - } - requestTime = new Date(); - - return [ - 404, - { - code: 1, - }, - ]; - }); + nock('http://fortest.com') + .get('/testRerty') + .times(config.requestRetry + 1) + .reply(function() { + requestRun(); + const now = new Date(); + if (requestTime) { + expect(now - requestTime).toBeGreaterThanOrEqual(100); + expect(now - requestTime).toBeLessThan(120); + } + requestTime = new Date(); + return [ + 404, + { + code: 1, + }, + ]; + }); try { - await axios.get('/test'); + await axios.get('http://fortest.com/testRerty'); } catch (error) { - expect(error.response.status).toBe(404); - expect(error.response.data.code).toBe(1); + expect(error.name).toBe('RequestError'); } // retries - expect(requestRun).toHaveBeenCalledTimes(config.requestRetry + 1); + expect(requestRun).toHaveBeenCalledTimes(config.requestRetry); }); });