feat: add 辽宁工程技术大学教务在线全文抓取

This commit is contained in:
ikvarxt
2021-01-10 10:56:55 +08:00
parent 8d570a14ae
commit d21161a91c
+62 -21
View File
@@ -1,26 +1,67 @@
// 以下代码参考 bupt/yz.js & bupt/utils.js 修改完成,在此感谢。
const got = require('@/utils/got');
const cheerio = require('cheerio');
const url = require('url');
module.exports = async (ctx) => {
const link = 'http://jwzx.lntu.edu.cn/index/jwgg.htm';
const response = await got.get(link);
const $ = cheerio.load(response.data);
const list = $('.tr-ri li').slice(0, 10);
module.exports = async(ctx) => {
const meta = {
selector: {
list: '.tr-ri ul',
item: 'li',
content: '.v_news_content',
},
url: 'http://jwzx.lntu.edu.cn/index/jwgg.htm',
}
const response = await got({
method: 'get',
url: `${meta.url}`,
});
const data = response.data;
ctx.state.data = {
title: '辽宁工程技术大学教务公告',
link: link,
description: '辽宁工程技术大学教务公告',
item:
list &&
list
.map((index, item) => {
item = $(item);
return {
title: item.find('li a').text(),
link: item.find('li a').attr('href'),
};
})
.get(),
};
const $ = cheerio.load(data);
const list = $(meta.selector.list).find(meta.selector.item).get();
const res = await ProcessFeed(list, meta);
ctx.state.data = {
title: '辽宁工程技术大学教务公告',
link: `${meta.url}`,
item: res,
};
};
const ProcessFeed = async(list, meta) =>
await Promise.all(
list.map(async(item) => {
let $ = cheerio.load(item);
// 通过解析过后的子项地址
const $url = url.resolve(meta.url, $('a').attr('href'));
// 加载内容页面
const response = await got({
method: 'get',
url: $url,
});
const data = response.data;
$ = cheerio.load(data); // 使用 cheerio 加载返回的 HTML
// 去除样式,好文明
$('img, div, span, p, table, td, tr').removeAttr('style');
$('style, script').remove();
// 截取掉「……教务处」
let title = $('title').text().split('-')[0];
// 列表上提取到的信息
return {
title: title,
description: $(meta.selector.content).html().replace(/<span*>(.*?)<\/span>/g, '$1').replace(/<span[^>]*>/g, '').replace(/<\/span[^>]*>/g, ''),
link: $url,
author: '辽宁工程技术大学教务处',
guid: $url, // 文章唯一标识
};
})
);