xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix：修复澎湃新闻频道抓取失败 (#4849)

This commit is contained in:
Wenhu 2020-05-26 11:27:12 +08:00 committed by GitHub
parent 24ebd2ee84
commit fe9b8fec3d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/routes/thepaper/channel.js
View File

@ -2,7 +2,7 @@ const utils = require('./utils');
module.exports = async (ctx) => {
const { id } = ctx.params;
const link = `https://m.thepaper.cn/list_${id}`;
const link = `https://www.thepaper.cn/channel_${id}`;
const items = await utils.ProcessFeed(link, ctx);
ctx.state.data = {

35
lib/routes/thepaper/utils.js
View File

@ -1,43 +1,38 @@
const cheerio = require('cheerio');
const dayjs = require('dayjs');
const url = require('url');
const got = require('@/utils/got');
module.exports = {
ProcessFeed: async (link, ctx) => {
const got_ins = got.extend({
headers: {
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A356 Safari/604.1',
},
});
const res = await got_ins.get(link);
const res = await got.get(link);
const $ = cheerio.load(res.data);
const list = $('p.news_tit01, div.news_tit02').slice(0, 10).get();
const list = $('#mainContent .news_li').slice(0, 10).get();
return await Promise.all(
list.map(async (item) => {
const $ = cheerio.load(item);
const itemUrl = url.resolve(link, $(item).find('a').attr('href'));
const itemUrlID = $(item).find('.news_tu a').attr('href');
const itemUrl = `https://www.thepaper.cn/${itemUrlID}`;
const cache = await ctx.cache.get(itemUrl);
if (cache) {
return Promise.resolve(JSON.parse(cache));
}
const res = await got_ins.get(itemUrl);
const res = await got.get(itemUrl);
const content = cheerio.load(res.data);
const serverOffset = new Date().getTimezoneOffset() / 60;
const author = content('.newscontent .news_about p').slice(0, 1).text();
content('.newscontent .news_about p span').remove();
const pubDateStr = content('.newscontent .news_about p').slice(1, 2).text().trim();
const single = {
title: $(item).find('a').text(),
title: content('.newscontent .news_title').text(),
guid: itemUrl,
link: itemUrl.replace('https://m.', 'https://'),
description: content('#v3cont_id > div.news_content > div.news_part_father > div > div:nth-child(1)').html(),
pubDate: content('#v3cont_id > div.news_content > p:nth-child(3)').html()
? dayjs(content('#v3cont_id > div.news_content > p:nth-child(3)').html().split(' ')[0])
.add(-8 - serverOffset, 'hour')
.toISOString()
: null,
author: content('#v3cont_id > div.news_content > p:nth-child(2)').text(),
link: itemUrl,
description: content('.newscontent .news_txt').html(),
pubDate: dayjs(pubDateStr)
.add(-8 - serverOffset, 'hour')
.toISOString(),
author: author,
};
return Promise.resolve(single);
})