xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(route): fix bjfu encoding (#13223) 

* fix(route): fix bjfu encoding

* fix: use cache.tryGet

---------
This commit is contained in:
JimenezLi 2023-09-07 04:34:35 +08:00 committed by GitHub
parent bb524fbb19
commit fa27a51ebf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 148 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
lib/v2/bjfu/grs.js
View File

@ -1,60 +1,46 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const iconv = require('iconv-lite');
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
module.exports = async (ctx) => {
const url = 'http://graduate.bjfu.edu.cn/pygl/pydt/index.html';
const response = await got.get(url, {
responseType: 'buffer',
});
const data = iconv.decode(response.data, 'gb2312');
const response = await got.get(url);
const data = response.data;
const $ = cheerio.load(data);
const list = $('.itemList li')
.slice(0, 10)
.map((i, e) => {
.slice(0, 11)
.toArray()
.map((e) => {
const element = $(e);
const title = element.find('li a').attr('title');
const link = element.find('li a').attr('href');
const date = new Date(
element
.find('li a')
.text()
.match(/\d{4}-\d{2}-\d{2}/)
);
const timeZone = 8;
const serverOffset = date.getTimezoneOffset() / 60;
const pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString();
const date = element
.find('li a')
.text()
.match(/\d{4}-\d{2}-\d{2}/);
const pubDate = timezone(parseDate(date), 8);
return {
title,
description: '',
link: 'http://graduate.bjfu.edu.cn/pygl/pydt/' + link,
author: '北京林业大学研究生院培养动态',
pubDate,
};
})
.get();
});
const result = await Promise.all(
list.map(async (item) => {
const link = item.link;
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const itemReponse = await got.get(item.link);
const data = itemReponse.data;
const itemElement = cheerio.load(data);
const cache = await ctx.cache.get(link);
if (cache) {
return Promise.resolve(JSON.parse(cache));
}
item.description = itemElement('.articleTxt').html();
const itemReponse = await got.get(link, {
responseType: 'buffer',
});
const data = iconv.decode(itemReponse.data, 'gb2312');
const itemElement = cheerio.load(data);
item.description = itemElement('.articleTxt').html();
ctx.cache.set(link, JSON.stringify(item));
return item;
})
return item;
})
)
);
ctx.state.data = {

16
lib/v2/bjfu/it/index.js
View File

@ -1,7 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const util = require('./utils');
const iconv = require('iconv-lite'); // 转码
const iconv = require('iconv-lite');
module.exports = async (ctx) => {
const type = ctx.params.type;
@ -27,16 +27,18 @@ module.exports = async (ctx) => {
const response = await got({
method: 'get',
responseType: 'buffer', // 转码
responseType: 'buffer',
url: base,
});
const data = iconv.decode(response.data, 'gb2312'); // 转码
const $ = cheerio.load(data);
const data = response.data;
let $ = cheerio.load(iconv.decode(data, 'utf-8'));
const charset = $('meta[charset]').attr('charset');
if (charset?.toLowerCase() !== 'utf-8') {
$ = cheerio.load(iconv.decode(data, charset ?? 'utf-8'));
}
// const list = $('div[item-content]').slice(0, 10).get();
const list = $('.item-content').get();
const list = $('.item-content').toArray();
const result = await util.ProcessFeed(base, list, ctx.cache); // 感谢@hoilc指导

67
lib/v2/bjfu/it/utils.js
View File

@ -1,62 +1,71 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const iconv = require('iconv-lite'); // 转码
const iconv = require('iconv-lite');
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
// 完整文章页
async function load(link) {
const response = await got.get(link, {
responseType: 'buffer',
});
let response;
try {
response = await got.get(link, {
responseType: 'buffer',
});
} catch (e) {
return { description: '' };
}
const data = iconv.decode(response.data, 'gb2312'); // 转码
const data = response.data; // 不用转码
// 加载文章内容
const $ = cheerio.load(data);
// 解析日期
const pubDate = timezone(
parseDate(
$('.template-head-info')
.text()
.match(/\d{4}-\d{2}-\d{2}/)
),
+8
);
let $ = cheerio.load(iconv.decode(data, 'utf-8'));
const charset = $('meta[charset]').attr('charset');
if (charset?.toLowerCase() !== 'utf-8') {
$ = cheerio.load(iconv.decode(data, charset ?? 'utf-8'));
}
// 提取内容
const description = $('.template-body').html();
const description = ($('.template-body').length ? $('.template-body').html() : '') + ($('.template-tail').length ? $('.template-tail').html() : '');
// 返回解析的结果
return { description, pubDate };
return { description };
}
const ProcessFeed = (base, list, caches) =>
// 使用 Promise.all() 进行 async 并发
Promise.all(
// 遍历每一篇文章
list.map(async (item) => {
list.map((item) => {
const $ = cheerio.load(item);
const $title = $('a');
// 还原相对链接为绝对链接
const itemUrl = new URL($title.attr('href'), base).href; // 感谢@hoilc指导
// 列表上提取到的信息
const single = {
title: $title.text(),
link: itemUrl,
author: '北林信息',
guid: itemUrl,
};
// 解析日期
const pubDate = timezone(
parseDate(
$('span')
.text()
.match(/\d{4}-\d{2}-\d{2}/)
),
+8
);
// 使用tryGet方法从缓存获取内容。
// 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
const other = await caches.tryGet(itemUrl, () => load(itemUrl));
return caches.tryGet(itemUrl, async () => {
const { description } = await load(itemUrl);
// 合并解析后的结果集作为该篇文章最终的输出结果
return { ...single, ...other };
// 列表上提取到的信息
return {
title: $title.text(),
link: itemUrl,
author: '北林信息',
description,
pubDate,
};
});
})
);
module.exports = {

11
lib/v2/bjfu/jwc/index.js
View File

@ -1,16 +1,11 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const util = require('./utils');
const iconv = require('iconv-lite'); // 转码
module.exports = async (ctx) => {
const type = ctx.params.type;
let title, path;
switch (type) {
case 'jwkx':
title = '教务快讯';
path = 'jwkx/';
break;
case 'jgdt':
title = '教改动态';
path = 'jgdt/';
@ -27,6 +22,7 @@ module.exports = async (ctx) => {
title = '图片新闻';
path = 'tpxw/';
break;
case 'jwkx':
default:
title = '教务快讯';
path = 'jwkx/';
@ -35,14 +31,13 @@ module.exports = async (ctx) => {
const response = await got({
method: 'get',
responseType: 'buffer', // 转码
url: base,
});
const data = iconv.decode(response.data, 'gb2312'); // 转码
const data = response.data; // 不用转码
const $ = cheerio.load(data);
const list = $('.list_c li').slice(0, 10).get();
const list = $('.list_c li').slice(0, 15).toArray();
const result = await util.ProcessFeed(base, list, ctx.cache); // 感谢@hoilc指导

53
lib/v2/bjfu/jwc/utils.js
View File

@ -1,61 +1,58 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const iconv = require('iconv-lite'); // 转码
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
// 完整文章页
async function load(link) {
const response = await got.get(link, {
responseType: 'buffer',
});
const response = await got.get(link);
const data = iconv.decode(response.data, 'gb2312'); // 转码
const data = response.data; // 不用转码
// 加载文章内容
const $ = cheerio.load(data);
// 解析日期
const pubDate = timezone(
parseDate(
$('div #con_djl')
.text()
.match(/\d{4}-\d{2}-\d{2}/)
),
+8
);
// 提取内容
const description = $('#con_c').html();
const description = ($('#con_c').length ? $('#con_c').html() : '') + ($('#con_fujian').length ? $('#con_fujian').html() : '');
// 返回解析的结果
return { description, pubDate };
return { description };
}
const ProcessFeed = (base, list, caches) =>
Promise.all(
// 遍历每一篇文章
list.map(async (item) => {
list.map((item) => {
const $ = cheerio.load(item);
const $title = $('a');
// 还原相对链接为绝对链接
const itemUrl = new URL($title.attr('href'), base).href; // 感谢@hoilc指导
// 列表上提取到的信息
const single = {
title: $title.text(),
link: itemUrl,
author: '北林教务处',
guid: itemUrl,
};
// 解析日期
const pubDate = timezone(
parseDate(
$('.datetime')
.text()
.match(/\d{4}-\d{2}-\d{2}/)
),
+8
);
// 使用tryGet方法从缓存获取内容。
// 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
const other = await caches.tryGet(itemUrl, () => load(itemUrl));
return caches.tryGet(itemUrl, async () => {
const { description } = await load(itemUrl);
// 合并解析后的结果集作为该篇文章最终的输出结果
return { ...single, ...other };
// 列表上提取到的信息
return {
title: $title.text(),
link: itemUrl,
author: '北林教务处',
description,
pubDate,
};
});
})
);
module.exports = {

59
lib/v2/bjfu/kjc.js
View File

@ -1,60 +1,47 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const iconv = require('iconv-lite');
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
module.exports = async (ctx) => {
const url = 'http://kyc.bjfu.edu.cn/tztg/index.html';
const response = await got.get(url, {
responseType: 'buffer',
});
const data = iconv.decode(response.data, 'gb2312');
const response = await got.get(url);
const data = response.data;
const $ = cheerio.load(data);
const list = $('.ll_con_r_b li')
.slice(0, 10)
.map((i, e) => {
.slice(0, 15)
.toArray()
.map((e) => {
const element = $(e);
const title = element.find('.ll_con_r_b_title a').text();
const link = element.find('a').attr('href');
const date = new Date(
element
.find('.ll_con_r_b_time')
.text()
.match(/\d{4}-\d{2}-\d{2}/)
);
const timeZone = 8;
const serverOffset = date.getTimezoneOffset() / 60;
const pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString();
const date = element
.find('.ll_con_r_b_time')
.text()
.match(/\d{4}-\d{2}-\d{2}/);
const pubDate = timezone(parseDate(date), 8);
return {
title,
description: '',
link: 'http://kyc.bjfu.edu.cn/tztg/' + link,
author: '北京林业大学科技处通知公告',
pubDate,
};
})
.get();
});
const result = await Promise.all(
list.map(async (item) => {
const link = item.link;
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const itemReponse = await got.get(item.link);
const data = itemReponse.data;
const itemElement = cheerio.load(data);
const cache = await ctx.cache.get(link);
if (cache) {
return Promise.resolve(JSON.parse(cache));
}
item.description = itemElement('#a_con_l_con').html();
item.title = item.title.includes('...') ? itemElement('#a_con_l_title').text() : item.title;
const itemReponse = await got.get(link, {
responseType: 'buffer',
});
const data = iconv.decode(itemReponse.data, 'gb2312');
const itemElement = cheerio.load(data);
item.description = itemElement('#a_con_l_con').html();
ctx.cache.set(link, JSON.stringify(item));
return item;
})
return item;
})
)
);
ctx.state.data = {

21
lib/v2/bjfu/news/index.js
View File

@ -1,16 +1,12 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const util = require('./utils');
const iconv = require('iconv-lite'); // 转码
const iconv = require('iconv-lite');
module.exports = async (ctx) => {
const type = ctx.params.type;
let title, path;
switch (type) {
case 'lsyw':
title = '绿色要闻';
path = 'lsyw/';
break;
case 'xydt':
title = '校园动态';
path = 'lsxy/';
@ -27,6 +23,7 @@ module.exports = async (ctx) => {
title = '一周排行';
path = 'yzph/';
break;
case 'lsyw':
default:
title = '绿色要闻';
path = 'lsyw/';
@ -35,14 +32,20 @@ module.exports = async (ctx) => {
const response = await got({
method: 'get',
responseType: 'buffer', // 转码
responseType: 'buffer',
url: base,
});
const data = iconv.decode(response.data, 'gb2312'); // 转码
const $ = cheerio.load(data);
const data = response.data;
let $ = cheerio.load(iconv.decode(data, 'utf-8'));
const charset = $('meta[http-equiv="Content-Type"]')
.attr('content')
.match(/charset=(.*)/)?.[1];
if (charset?.toLowerCase() !== 'utf-8') {
$ = cheerio.load(iconv.decode(data, charset ?? 'utf-8'));
}
const list = $('.news_ul li').slice(0, 10).get();
const list = $('.news_ul li').slice(0, 12).toArray();
const result = await util.ProcessFeed(base, list, ctx.cache); // 感谢@hoilc指导

34
lib/v2/bjfu/news/utils.js
View File

@ -1,16 +1,13 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const iconv = require('iconv-lite'); // 转码
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
// 完整文章页
async function load(link) {
const response = await got.get(link, {
responseType: 'buffer',
});
const response = await got.get(link);
const data = iconv.decode(response.data, 'gb2312'); // 转码
const data = response.data; // 不用转码
// 加载文章内容
const $ = cheerio.load(data);
@ -27,35 +24,36 @@ async function load(link) {
// 提取内容
const description = $('.article_con').html();
const title = $('h2').text();
// 返回解析的结果
return { description, pubDate };
return { description, pubDate, title };
}
const ProcessFeed = (base, list, caches) =>
Promise.all(
// 遍历每一篇文章
list.map(async (item) => {
list.map((item) => {
const $ = cheerio.load(item);
const $title = $('a');
// 还原相对链接为绝对链接
const itemUrl = new URL($title.attr('href'), base).href; // 感谢@hoilc指导
// 列表上提取到的信息
const single = {
title: $title.text(),
link: itemUrl,
author: '绿色新闻网',
guid: itemUrl,
};
// 使用tryGet方法从缓存获取内容。
// 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
const other = await caches.tryGet(itemUrl, () => load(itemUrl));
return caches.tryGet(itemUrl, async () => {
const { description, pubDate, title } = await load(itemUrl);
// 合并解析后的结果集作为该篇文章最终的输出结果
return { ...single, ...other };
// 列表上提取到的信息
return {
title: $title.text().includes('...') ? title : $title.text(),
link: itemUrl,
author: '绿色新闻网',
description,
pubDate,
};
});
})
);
module.exports = {