diff --git a/docs/university.md b/docs/university.md index c78677d63..dcb5e5326 100644 --- a/docs/university.md +++ b/docs/university.md @@ -536,17 +536,17 @@ category 列表： ### 教务通知 - + -| 全部 | 教学服务 | 教学建设 | 学生培养 | 教学资源 | -| ---- | -------- | -------- | -------- | -------- | -| all | jxfw | jxjs | xspy | jxzy | +| 教学服务 | 教学建设 | 学生培养 | 教学资源 | +| ------------- | -------- | -------- | -------- | +| jxfw(default) | jxjs | xspy | jxzy | ### 计算机科学与技术学院 - + | 通知公告 | 新闻动态 | 科研动态 | 教学动态 | 学生工作 | 招生信息 | 就业信息 | | -------- | -------- | -------- | -------- | -------- | -------- | -------- | @@ -556,7 +556,7 @@ category 列表： ### 研究生院 - + | 最近动态 | 研院新闻 | 上级文件 | 管理文件 | 信息服务 | | -------- | -------- | -------- | -------- | -------- | diff --git a/lib/routes/universities/nuaa/cs/index.js b/lib/routes/universities/nuaa/cs/index.js index ef6e75540..598607680 100644 --- a/lib/routes/universities/nuaa/cs/index.js +++ b/lib/routes/universities/nuaa/cs/index.js @@ -1,7 +1,7 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const url = require('url'); - +const getCookie = require('../utils/pypasswaf'); const host = 'http://cs.nuaa.edu.cn/'; const map = new Map([ @@ -19,21 +19,27 @@ module.exports = async (ctx) => { const suffix = map.get(type).suffix; const link = url.resolve(host, suffix); - const response = await got.get(link); + const cookie = await getCookie(); + const gotConfig = { + headers: { + cookie, + }, + }; + const response = await got.get(link, gotConfig); const $ = cheerio.load(response.data); - const list = $('tr.section') + const list = $('#news_list > ul > li') .slice(0, 10) .map(function() { const info = { title: $(this) - .find('td:nth-child(2) a') + .find('a') .attr('title'), link: $(this) - .find('td:nth-child(2) a') + .find('a') .attr('href'), date: $(this) - .find('td:nth-child(3)') + .find('span') .text(), }; return info; @@ -56,7 +62,7 @@ module.exports = async (ctx) => { let description = itemUrl; if (pageType === 'htm' || pageType === 'html') { - const response = await got.get(itemUrl); + const response = await got.get(itemUrl, gotConfig); const $ = cheerio.load(response.data); description = $('.wp_articlecontent') .html() diff --git a/lib/routes/universities/nuaa/jwc/jwc.js b/lib/routes/universities/nuaa/jwc/jwc.js index d84ce30c4..766f63bde 100644 --- a/lib/routes/universities/nuaa/jwc/jwc.js +++ b/lib/routes/universities/nuaa/jwc/jwc.js @@ -2,17 +2,26 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const url = require('url'); const host = 'http://aao.nuaa.edu.cn/'; +// https is not allowed in nuaa;interesting ha const map = { - all: 0, - jxfw: 333, - xspy: 334, - jxjs: 336, - jxzy: 334, + default: 8230, + jxfw: 8230, // 教学服务 + xspy: 8231, // 学生培养 + jxjs: 8232, // 教学建设 + jxzy: 8233, // 教学资源 }; -async function load(link) { - const response = await got.get(host + link); +async function load(link, cookie, ctx) { + const cache = await ctx.cache.get(link); + if (cache) { + return JSON.parse(cache); + } + const response = await got.get(link, { + headers: { + cookie, + }, + }); const $ = cheerio.load(response.data); const pubDate = new Date( $('.release-time') @@ -24,29 +33,66 @@ async function load(link) { for (let k = 0; k < images.length; k++) { $(images[k]).replaceWith(``); } - const description = $('.detail-div').html(); - return { pubDate, description }; + const description = $('.wp_articlecontent').html(); + const result = { pubDate, description }; + ctx.cache.set(link, JSON.stringify(result)); + + return result; } module.exports = async (ctx) => { - const type = ctx.params.type || 'all'; - const response = await got({ - method: 'get', - url: host + 'index_sub/notice/' + map[type], + const browser = await require('@/utils/puppeteer')(); + const type = ctx.params.type || 'default'; + const listUrl = `${host}${map[type]}/list.htm`; + + const page = await browser.newPage(); + await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'); + await page.evaluateOnNewDocument(() => { + // eslint-disable-next-line + Object.defineProperty(navigator, 'webdriver', { get: () => false }); }); - const $ = cheerio.load(response.data); - const list = $('.inform-content a').get(); - const process = await Promise.all( - list.map(async (item) => { - item = $(item); - const itemUrl = item.attr('onclick').slice(13, -3); - const single = { - title: item.text(), - link: url.resolve(host, itemUrl), - guid: url.resolve(host, itemUrl), + // 南航的新waf现在采用cookie验证的方法防止爬虫，并且会对chrome headless进行检测 + + await page.goto(listUrl, { + waitUntil: 'networkidle0', + }); + + let msg; + try { + // data = await page.$('ul.right-ul') + msg = await page.$$eval('ul.right-ul > li', (es) => + es.map((e) => { + const html = e.innerHTML; + + let [, title] = html.match(/(.+)<\/a>/); + if (title.match(/font/)) { + [, title] = title.match(/>(.+)(.+?) `${name}=${value}`).join('; '); + browser.close(); + msg = await Promise.all( + msg.map(async (e) => { + const link = url.resolve(host, e.link); + return { + ...e, + link, + guid: link, + ...(await load(link, cookie, ctx)), }; - const other = await load(itemUrl); - return Promise.resolve(Object.assign({}, single, other)); }) ); @@ -54,6 +100,6 @@ module.exports = async (ctx) => { title: '南航教务', link: host, description: '南航教务RSS', - item: process, + item: msg, }; }; diff --git a/lib/routes/universities/nuaa/utils/pypasswaf.js b/lib/routes/universities/nuaa/utils/pypasswaf.js new file mode 100644 index 000000000..1759260c6 --- /dev/null +++ b/lib/routes/universities/nuaa/utils/pypasswaf.js @@ -0,0 +1,24 @@ +const host = 'http://aao.nuaa.edu.cn/'; + +/** + * async function 获取cookie + * @desc 返回一个可用的cookie，使用 `got` 发起请求的时候，传入到`options.header.cookie`即可 + */ +module.exports = async function getCookie() { + const browser = await require('@/utils/puppeteer')(); + const page = await browser.newPage(); + await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'); + await page.evaluateOnNewDocument(() => { + // eslint-disable-next-line + Object.defineProperty(navigator, 'webdriver', { get: () => false }); + }); + + await page.goto(host, { + waitUntil: 'networkidle0', + }); + + let cookie = await page.cookies(); + cookie = cookie.map(({ name, value }) => `${name}=${value}`).join('; '); + await browser.close(); + return cookie; +}; diff --git a/lib/routes/universities/nuaa/yjsy/yjsy.js b/lib/routes/universities/nuaa/yjsy/yjsy.js index 2ed641bbb..8c3fa71c2 100644 --- a/lib/routes/universities/nuaa/yjsy/yjsy.js +++ b/lib/routes/universities/nuaa/yjsy/yjsy.js @@ -2,6 +2,7 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const url = require('url'); const host = 'http://www.graduate.nuaa.edu.cn/'; +const getCookie = require('../utils/pypasswaf'); const map = { latest: '2146/list.htm', @@ -11,12 +12,12 @@ const map = { xxfw: '2147/list.htm', }; -async function load(link, ctx) { +async function load(link, ctx, gotConfig) { const cache = await ctx.cache.get(link); if (cache) { return cache; } - const response = await got.get(host + link); + const response = await got.get(host + link, gotConfig); const $ = cheerio.load(response.data); const images = $('img'); for (let k = 0; k < images.length; k++) { @@ -28,11 +29,21 @@ async function load(link, ctx) { } module.exports = async (ctx) => { + const cookie = await getCookie(); + + const gotConfig = { + headers: { + cookie, + }, + }; const type = ctx.params.type || 'latest'; - const response = await got({ - method: 'get', - url: host + map[type], - }); + const response = await got( + { + method: 'get', + url: host + map[type], + }, + gotConfig + ); const $ = cheerio.load(response.data); const list = $('#news_list > tbody > tr > td > table > tbody > tr:nth-child(1)') .slice(0, 10) @@ -48,7 +59,7 @@ module.exports = async (ctx) => { guid: url.resolve(host, itemUrl), pubDate: t.text().slice(0, 10), }; - const other = await load(itemUrl, ctx); + const other = await load(itemUrl, ctx, gotConfig); return Promise.resolve(Object.assign({}, single, other)); }) );