xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(route)(wechat/ce): anti-crawler and pubDate (#9495) 

Signed-off-by: Rongrong <15956627+Rongronggg9@users.noreply.github.com>
This commit is contained in:
Rongrong 2022-04-09 00:00:02 +08:00 committed by GitHub
parent ef94bcde8e
commit 2dab2ad5bc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 45 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
lib/routes/tencent/wechat/ce.js
View File

@ -1,33 +1,62 @@
const parser = require('@/utils/rss-parser');
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { fixArticleContent } = require('@/utils/wechat-mp');
const { parseDate } = require('@/utils/parse-date');
// any UA containing "RSS" can pass the check
// mark the UA as a desktop UA with "(X11; Linux x86_64)"
const UA = 'Mozilla/5.0 (X11; Linux x86_64) RSS Reader';
module.exports = async (ctx) => {
const { id } = ctx.params;
const feed = await parser.parseURL(`https://posts.careerengine.us/author/${id}/rss`);
const feed = await parser.parseString(
await got
.get(`https://posts.careerengine.us/author/${id}/rss`, {
headers: {
'User-Agent': UA,
},
})
.then((_) => _.data)
);
const items = await Promise.all(
feed.items.splice(0, 10).map(async (item) => {
const response = await got.get(item.link);
// generally speaking, changing `item.link` of an existing route could potentially break `item.guid`
// but since the route has been down for at least 8 months, it's probably safe
item.link = item.link.replace(/^http:\/\//, 'https://');
return await ctx.cache.tryGet(item.link, async () => {
const response = await got.get(item.link, {
headers: {
'User-Agent': UA,
},
});
const $ = cheerio.load(response.data);
const post = $('.post');
const $ = cheerio.load(response.data);
post.find('img').each((_, img) => {
const dataSrc = $(img).attr('data-src');
if (dataSrc) {
$(img).attr('src', dataSrc);
const description = fixArticleContent($('.post'));
let pubDate = item.pubDate;
if (!pubDate || pubDate === 'Invalid Date') {
// sometimes the pubDate is not available in the official feed
const postDate = $('.post-date')
.text()
.replace(/\s+|发表/g, '');
// the date format is "发表 YYYY年MM月DD日 "
// following the official feed behavior: imprecise date is in UTC
// `<pubDate>Mon, 04 Apr 2022 00:00:00 GMT</pubDate>`
pubDate = parseDate(postDate, 'YYYY年MM月DD日');
pubDate = new Date(pubDate.getTime() - pubDate.getTimezoneOffset() * 60 * 1000);
}
});
const single = {
title: item.title,
description: post.html(),
pubDate: item.pubDate,
link: item.link,
};
return Promise.resolve(single);
return {
title: item.title,
description,
pubDate,
link: item.link,
};
});
})
);