fix: 网站改版,使用JS加密;通过puppeteer加载页面

This commit is contained in:
jinkin 2021-01-12 20:24:34 +08:00
parent f87feca619
commit 323029c42d
1 changed files with 44 additions and 5 deletions

View File

@ -1,10 +1,48 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
module.exports = async (ctx) => {
const baseIndexUrl = 'https://www.sice.uestc.edu.cn/';
const response = await got.get(baseIndexUrl);
const $ = cheerio.load(response.data);
const baseIndexUrl = 'https://www.sice.uestc.edu.cn/index.htm';
const browser = await require('@/utils/puppeteer')();
const page = await browser.newPage();
// 浏览器伪装
await page.evaluateOnNewDocument(() => { //在每个新页面打开前执行以下脚本
const newProto = navigator.__proto__;
delete newProto.webdriver; //删除navigator.webdriver字段
navigator.__proto__ = newProto;
window.chrome = {}; //添加window.chrome字段为增加真实性还需向内部填充一些值
window.chrome.app = {"InstallState":"hehe", "RunningState":"haha", "getDetails":"xixi", "getIsInstalled":"ohno"};
window.chrome.csi = function(){};
window.chrome.loadTimes = function(){};
window.chrome.runtime = function(){};
Object.defineProperty(navigator, 'userAgent', { //userAgent在无头模式下有headless字样所以需覆写
get: () => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36",
});
Object.defineProperty(navigator, 'plugins', { //伪装真实的插件信息
get: () => [{"description": "Portable Document Format",
"filename": "internal-pdf-viewer",
"length": 1,
"name": "Chrome PDF Plugin"}]
});
Object.defineProperty(navigator, 'languages', { //添加语言
get: () => ["zh-CN", "zh", "en"],
});
const originalQuery = window.navigator.permissions.query; //notification伪装
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
});
await page.goto(baseIndexUrl, {
waitUntil: "networkidle0"
});
const content = await page.content();
await browser.close();
const $ = cheerio.load(content);
const out = $('.notice p').map((index, item) => {
item = $(item);
let date = new Date(new Date().getFullYear() + '-' + item.find('a.date').text());
@ -18,8 +56,9 @@ module.exports = async (ctx) => {
};
}
).get();
// console.log(out);
ctx.state.data = {
title: '信通通知公告',
link: 'https://www.sice.uestc.edu.cn/tzgg/yb.htm',