From ef86f5b105ac7d47f99a866768a9cbcfec52ca82 Mon Sep 17 00:00:00 2001 From: Artin Date: Thu, 30 Apr 2020 16:06:26 +0800 Subject: [PATCH] feat(swust): follow meta redirect (#4607) --- lib/routes/universities/swust/cs.js | 26 +++--------- lib/routes/universities/swust/helper.js | 47 +++++++++++++++++++++ lib/routes/universities/swust/jwc_news.js | 15 ++----- lib/routes/universities/swust/jwc_notice.js | 17 ++------ 4 files changed, 61 insertions(+), 44 deletions(-) create mode 100644 lib/routes/universities/swust/helper.js diff --git a/lib/routes/universities/swust/cs.js b/lib/routes/universities/swust/cs.js index ad967dba6..ec0a5afc6 100644 --- a/lib/routes/universities/swust/cs.js +++ b/lib/routes/universities/swust/cs.js @@ -1,13 +1,8 @@ -const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { instance } = require('./helper'); const host = 'http://www.cs.swust.edu.cn/'; module.exports = async (ctx) => { - const got_ins = got.extend({ - headers: { - Referer: host, - }, - }); const type = ctx.params.type || 1; let info = '新闻动态'; let word = 'news'; @@ -22,8 +17,8 @@ module.exports = async (ctx) => { word = 'Teach_Research'; } - const response = await got_ins.get(host + word); - const $ = cheerio.load(response.data); + const response = await instance.get(host + word); + const $ = cheerio.load(response.body); const text = $('a', '.list.list3'); const resultItems = await Promise.all( text.toArray().map(async (item) => { @@ -36,20 +31,14 @@ module.exports = async (ctx) => { if (value) { resultItem = JSON.parse(value); } else { - const article = await got_ins.get(link); - const $1 = cheerio.load(article.data); - const res = $1('small') + const date = $('.imgdesc', $item) .contents() .filter(function () { return this.nodeType === 3; }) - .text(); - const reg = new RegExp('\n', 'g'); // 匹配回车符 - const str = res.replace(reg, '').toString(); // 替换回车符为""，再转为string类型 - const arr = str.split(' '); // 将字符串每个字符转换为数组的元素 - let date = arr[1] + ' ' + arr[2]; - date = date.replace(/^\s+|\s+$/g, ''); - + .text() + .replace(']', '') + .replace('[', ''); resultItem = { title: $('.imgdesc', $item).find('.imgtitle').text(), description: $('.imgdesc', $item).find('.imgtext').text(), @@ -59,7 +48,6 @@ module.exports = async (ctx) => { ctx.cache.set(link, JSON.stringify(resultItem)); } - // }; return Promise.resolve(resultItem); }) ); diff --git a/lib/routes/universities/swust/helper.js b/lib/routes/universities/swust/helper.js new file mode 100644 index 000000000..85db45443 --- /dev/null +++ b/lib/routes/universities/swust/helper.js @@ -0,0 +1,47 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +function evil(fn) { + const Fn = Function; // 一个变量指向Function，防止有些前端编译工具报错 + return new Fn('return ' + fn)(); +} +const userAgent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'; + +const instance = got.extend({ + headers: { + 'User-Agent': userAgent, + }, + hooks: { + afterResponse: [ + (response) => { + try { + JSON.parse(response.body); + return response; + } catch (e) { + const $ = cheerio.load(response.body); + const redirect = $('meta[http-equiv="refresh"]'); + if (!(redirect.length > 0)) { + return response; + } + let [, url] = redirect.attr('content').split(';'); + url = url.trim(); + if (url.startsWith('url=')) { + url = url.slice(4); + } + + return instance.get({ + url: url, + headers: { + Referer: response.url, + }, + }); + } + }, + ], + }, + mutableDefaults: true, +}); + +module.exports = { + instance, + evil, +}; diff --git a/lib/routes/universities/swust/jwc_news.js b/lib/routes/universities/swust/jwc_news.js index 2bf8a1135..44344a9e6 100644 --- a/lib/routes/universities/swust/jwc_news.js +++ b/lib/routes/universities/swust/jwc_news.js @@ -1,21 +1,12 @@ -const got = require('@/utils/got'); - -// 转码需要设定返回数据的格式，其可选项是arraybuffer,blob,document,json,text,stream -// 默认为json -const got_ins = got.extend({ responseType: 'text' }); +const { instance, evil } = require('./helper'); const api = 'http://www.dean.swust.edu.cn/cms/portal/news.json'; const host = 'http://www.dean.swust.edu.cn/'; const page = 'http://www.dean.swust.edu.cn/news/'; -function evil(fn) { - const Fn = Function; // 一个变量指向Function，防止有些前端编译工具报错 - return new Fn('return ' + fn)(); -} - module.exports = async (ctx) => { - const response = await got_ins.get(api); - const data = evil(response.data); + const response = await instance.get(api); + const data = evil(response.body); const resultItems = await Promise.all( data.map(async (item) => { diff --git a/lib/routes/universities/swust/jwc_notice.js b/lib/routes/universities/swust/jwc_notice.js index c5b6f1327..1e4f51f53 100644 --- a/lib/routes/universities/swust/jwc_notice.js +++ b/lib/routes/universities/swust/jwc_notice.js @@ -1,24 +1,15 @@ -const got = require('@/utils/got'); - -// 转码需要设定返回数据的格式，其可选项是arraybuffer,blob,document,json,text,stream -// 默认为json -const got_ins = got.extend({ responseType: 'text' }); +const { instance, evil } = require('./helper'); const api = 'http://www.dean.swust.edu.cn/cms/portal/notice.json'; const page = 'http://www.dean.swust.edu.cn/notice/page/'; const tag = 'http://www.dean.swust.edu.cn/notice/tag/'; -function evil(fn) { - const Fn = Function; // 一个变量指向Function，防止有些前端编译工具报错 - return new Fn('return ' + fn)(); -} - module.exports = async (ctx) => { const type = ctx.params.type || 1; - const response = await got_ins.get(api); - const responsedata = evil(response.data); - const data = responsedata[type - 1]; + const response = await instance.get(api); + const responseData = evil(response.body); + const data = responseData[type - 1]; const info = data.name || ''; const tag_url = data.id || '';