fix: update pku.ss hosts due to the host change on November 2022 (#11784)

* update hosts due to the host change on November 2022

* refactor: combine duplicated function

---------
This commit is contained in:
Yuri Kiriyama 2023-02-06 19:05:26 +08:00 committed by GitHub
parent 6ace94fcd9
commit cc8c8697f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 49 additions and 126 deletions

View File

@ -1,48 +1,10 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const { baseUrl, getSingleRecord, getArticle } = require('./common');
const link = 'http://www.ss.pku.edu.cn';
const host = 'http://www.ss.pku.edu.cn/index.php/admission/admnotice';
const getSingleRecord = async () => {
const res = await got(host);
const $ = cheerio.load(res.data);
const list = $('#info-list-ul').find('li');
return (
list &&
list
.map((index, item) => {
item = $(item);
const date = item.find('.time').text();
return {
title: item.find('a').attr('title'),
pubDate: parseDate(date),
link: link + item.find('a').attr('href'),
};
})
.get()
);
};
const host = `${baseUrl}/admission/admnotice/`;
module.exports = async (ctx) => {
const items = await getSingleRecord();
const out = await Promise.all(
items.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const response = await got(item.link);
const $ = cheerio.load(response.data);
return {
title: item.title,
link: item.link,
description: $('.article-content').html(),
pubDate: item.pubDate,
};
})
)
);
const items = await getSingleRecord(host);
const out = await Promise.all(items.map((item) => getArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: '北大软微-招生通知',

37
lib/v2/pku/ss/common.js Normal file
View File

@ -0,0 +1,37 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const baseUrl = 'https://www.ss.pku.edu.cn';
const getSingleRecord = async (url) => {
const res = await got(url);
const $ = cheerio.load(res.data);
const list = $('#info-list-ul').find('li');
return list.toArray().map((item) => {
item = $(item);
const date = item.find('.time').text();
return {
title: item.find('a').attr('title'),
pubDate: parseDate(date),
link: baseUrl + item.find('a').attr('href'),
};
});
};
const getArticle = (item, tryGet) =>
tryGet(item.link, async () => {
const response = await got(item.link);
const $ = cheerio.load(response.data);
item.description = $('.article-content').html();
return item;
});
module.exports = {
baseUrl,
getSingleRecord,
getArticle,
};

View File

@ -1,48 +1,10 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const { baseUrl, getSingleRecord, getArticle } = require('./common');
const link = 'http://www.ss.pku.edu.cn';
const host = 'http://www.ss.pku.edu.cn/index.php/newscenter/notice';
const getSingleRecord = async () => {
const res = await got(host);
const $ = cheerio.load(res.data);
const list = $('#info-list-ul').find('li');
return (
list &&
list
.map((index, item) => {
item = $(item);
const date = item.find('.time').text();
return {
title: item.find('a').attr('title'),
pubDate: parseDate(date),
link: link + item.find('a').attr('href'),
};
})
.get()
);
};
const host = `${baseUrl}/newscenter/notice/`;
module.exports = async (ctx) => {
const items = await getSingleRecord();
const out = await Promise.all(
items.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const response = await got(item.link);
const $ = cheerio.load(response.data);
return {
title: item.title,
link: item.link,
description: $('.article-content').html(),
pubDate: item.pubDate,
};
})
)
);
const items = await getSingleRecord(host);
const out = await Promise.all(items.map((item) => getArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: '北大软微-通知公告',

View File

@ -1,48 +1,10 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const { baseUrl, getSingleRecord, getArticle } = require('./common');
const link = 'http://www.ss.pku.edu.cn';
const host = 'http://www.ss.pku.edu.cn/index.php/admission/admbrochure/admission01';
const getSingleRecord = async () => {
const res = await got(host);
const $ = cheerio.load(res.data);
const list = $('#info-list-ul').find('li');
return (
list &&
list
.map((index, item) => {
item = $(item);
const date = item.find('.time').text();
return {
title: item.find('a').attr('title'),
pubDate: parseDate(date),
link: link + item.find('a').attr('href'),
};
})
.get()
);
};
const host = `${baseUrl}/admission/admbrochure/`;
module.exports = async (ctx) => {
const items = await getSingleRecord();
const out = await Promise.all(
items.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const response = await got(item.link);
const $ = cheerio.load(response.data);
return {
title: item.title,
link: item.link,
description: $('.article-content').html(),
pubDate: item.pubDate,
};
})
)
);
const items = await getSingleRecord(host);
const out = await Promise.all(items.map((item) => getArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: '北大软微-硕士统考招生',