xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: nuaa/jwc (#3337)

This commit is contained in:
Seiry Yu 2019-11-19 17:16:46 +08:00 committed by DIYgod
parent 677c42e7b2
commit 9bb50dbc05
5 changed files with 133 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/university.md
View File

@ -536,17 +536,17 @@ category 列表：
### 教务通知
<Route author="arcosx" example="/nuaa/jwc/all" path="/universities/nuaa/jwc/:type" :paramsDesc="['分类名']">
<Route author="arcosx Seiry" example="/nuaa/jwc/default" path="/universities/nuaa/jwc/:type" :paramsDesc="['分类名']">
| 全部 | 教学服务 | 教学建设 | 学生培养 | 教学资源 |
| ---- | -------- | -------- | -------- | -------- |
| all | jxfw | jxjs | xspy | jxzy |
| 教学服务 | 教学建设 | 学生培养 | 教学资源 |
| ------------- | -------- | -------- | -------- |
| jxfw(default) | jxjs | xspy | jxzy |
</Route>
### 计算机科学与技术学院
<Route author="LogicJake" example="/nuaa/cs/kydt" path="/universities/nuaa/cs/:type?" :paramsDesc="['分类名']">
<Route author="LogicJake Seiry" example="/nuaa/cs/kydt" path="/universities/nuaa/cs/:type?" :paramsDesc="['分类名']"/>
| 通知公告 | 新闻动态 | 科研动态 | 教学动态 | 学生工作 | 招生信息 | 就业信息 |
| -------- | -------- | -------- | -------- | -------- | -------- | -------- |
@ -556,7 +556,7 @@ category 列表：
### 研究生院
<Route author="junfengP" example="/nuaa/yjsy/latest" path="/universities/nuaa/yjsy/:type?" :paramsDesc="['分类名']">
<Route author="junfengP Seiry" example="/nuaa/yjsy/latest" path="/universities/nuaa/yjsy/:type?" :paramsDesc="['分类名']"/>
| 最近动态 | 研院新闻 | 上级文件 | 管理文件 | 信息服务 |
| -------- | -------- | -------- | -------- | -------- |

20
lib/routes/universities/nuaa/cs/index.js
View File

@ -1,7 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const url = require('url');
const getCookie = require('../utils/pypasswaf');
const host = 'http://cs.nuaa.edu.cn/';
const map = new Map([
@ -19,21 +19,27 @@ module.exports = async (ctx) => {
const suffix = map.get(type).suffix;
const link = url.resolve(host, suffix);
const response = await got.get(link);
const cookie = await getCookie();
const gotConfig = {
headers: {
cookie,
},
};
const response = await got.get(link, gotConfig);
const $ = cheerio.load(response.data);
const list = $('tr.section')
const list = $('#news_list > ul > li')
.slice(0, 10)
.map(function() {
const info = {
title: $(this)
.find('td:nth-child(2) a')
.find('a')
.attr('title'),
link: $(this)
.find('td:nth-child(2) a')
.find('a')
.attr('href'),
date: $(this)
.find('td:nth-child(3)')
.find('span')
.text(),
};
return info;
@ -56,7 +62,7 @@ module.exports = async (ctx) => {
let description = itemUrl;
if (pageType === 'htm' || pageType === 'html') {
const response = await got.get(itemUrl);
const response = await got.get(itemUrl, gotConfig);
const $ = cheerio.load(response.data);
description = $('.wp_articlecontent')
.html()

98
lib/routes/universities/nuaa/jwc/jwc.js
View File

@ -2,17 +2,26 @@ const got = require('@/utils/got');
const cheerio = require('cheerio');
const url = require('url');
const host = 'http://aao.nuaa.edu.cn/';
// https is not allowed in nuaa;interesting ha
const map = {
all: 0,
jxfw: 333,
xspy: 334,
jxjs: 336,
jxzy: 334,
default: 8230,
jxfw: 8230, // 教学服务
xspy: 8231, // 学生培养
jxjs: 8232, // 教学建设
jxzy: 8233, // 教学资源
};
async function load(link) {
const response = await got.get(host + link);
async function load(link, cookie, ctx) {
const cache = await ctx.cache.get(link);
if (cache) {
return JSON.parse(cache);
}
const response = await got.get(link, {
headers: {
cookie,
},
});
const $ = cheerio.load(response.data);
const pubDate = new Date(
$('.release-time')
@ -24,29 +33,66 @@ async function load(link) {
for (let k = 0; k < images.length; k++) {
$(images[k]).replaceWith(`<img src="${url.resolve(host, $(images[k]).attr('src'))}" />`);
}
const description = $('.detail-div').html();
return { pubDate, description };
const description = $('.wp_articlecontent').html();
const result = { pubDate, description };
ctx.cache.set(link, JSON.stringify(result));
return result;
}
module.exports = async (ctx) => {
const type = ctx.params.type || 'all';
const response = await got({
method: 'get',
url: host + 'index_sub/notice/' + map[type],
const browser = await require('@/utils/puppeteer')();
const type = ctx.params.type || 'default';
const listUrl = `${host}${map[type]}/list.htm`;
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36');
await page.evaluateOnNewDocument(() => {
// eslint-disable-next-line
Object.defineProperty(navigator, 'webdriver', { get: () => false });
});
const $ = cheerio.load(response.data);
const list = $('.inform-content a').get();
const process = await Promise.all(
list.map(async (item) => {
item = $(item);
const itemUrl = item.attr('onclick').slice(13, -3);
const single = {
title: item.text(),
link: url.resolve(host, itemUrl),
guid: url.resolve(host, itemUrl),
// 南航的新waf现在采用cookie验证的方法防止爬虫，并且会对chrome headless进行检测
await page.goto(listUrl, {
waitUntil: 'networkidle0',
});
let msg;
try {
// data = await page.$('ul.right-ul')
msg = await page.$$eval('ul.right-ul > li', (es) =>
es.map((e) => {
const html = e.innerHTML;
let [, title] = html.match(/<a.+?>(.+)<\/a>/);
if (title.match(/font/)) {
[, title] = title.match(/>(.+)</);
}
const [, href] = html.match(/href="(.+?)"/);
// const [, time] = html.match(/-time">(.+?)</);
return {
title,
link: href,
guid: href,
};
})
);
} catch (e) {
// debugger
}
let cookie = await page.cookies();
cookie = cookie.map(({ name, value }) => `${name}=${value}`).join('; ');
browser.close();
msg = await Promise.all(
msg.map(async (e) => {
const link = url.resolve(host, e.link);
return {
...e,
link,
guid: link,
...(await load(link, cookie, ctx)),
};
const other = await load(itemUrl);
return Promise.resolve(Object.assign({}, single, other));
})
);
@ -54,6 +100,6 @@ module.exports = async (ctx) => {
title: '南航教务',
link: host,
description: '南航教务RSS',
item: process,
item: msg,
};
};

24
lib/routes/universities/nuaa/utils/pypasswaf.js Normal file
View File

@ -0,0 +1,24 @@
const host = 'http://aao.nuaa.edu.cn/';
/**
* async function 获取cookie
* @desc 返回一个可用的cookie，使用 `got` 发起请求的时候，传入到`options.header.cookie`即可
*/
module.exports = async function getCookie() {
const browser = await require('@/utils/puppeteer')();
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36');
await page.evaluateOnNewDocument(() => {
// eslint-disable-next-line
Object.defineProperty(navigator, 'webdriver', { get: () => false });
});
await page.goto(host, {
waitUntil: 'networkidle0',
});
let cookie = await page.cookies();
cookie = cookie.map(({ name, value }) => `${name}=${value}`).join('; ');
await browser.close();
return cookie;
};

25
lib/routes/universities/nuaa/yjsy/yjsy.js
View File

@ -2,6 +2,7 @@ const got = require('@/utils/got');
const cheerio = require('cheerio');
const url = require('url');
const host = 'http://www.graduate.nuaa.edu.cn/';
const getCookie = require('../utils/pypasswaf');
const map = {
latest: '2146/list.htm',
@ -11,12 +12,12 @@ const map = {
xxfw: '2147/list.htm',
};
async function load(link, ctx) {
async function load(link, ctx, gotConfig) {
const cache = await ctx.cache.get(link);
if (cache) {
return cache;
}
const response = await got.get(host + link);
const response = await got.get(host + link, gotConfig);
const $ = cheerio.load(response.data);
const images = $('img');
for (let k = 0; k < images.length; k++) {
@ -28,11 +29,21 @@ async function load(link, ctx) {
}
module.exports = async (ctx) => {
const cookie = await getCookie();
const gotConfig = {
headers: {
cookie,
},
};
const type = ctx.params.type || 'latest';
const response = await got({
method: 'get',
url: host + map[type],
});
const response = await got(
{
method: 'get',
url: host + map[type],
},
gotConfig
);
const $ = cheerio.load(response.data);
const list = $('#news_list > tbody > tr > td > table > tbody > tr:nth-child(1)')
.slice(0, 10)
@ -48,7 +59,7 @@ module.exports = async (ctx) => {
guid: url.resolve(host, itemUrl),
pubDate: t.text().slice(0, 10),
};
const other = await load(itemUrl, ctx);
const other = await load(itemUrl, ctx, gotConfig);
return Promise.resolve(Object.assign({}, single, other));
})
);