掘金分类改为全文RSS,一次获取文章数改为5 (#444)

掘金的分类文章中站内文章获取全文,非站内文章保持摘要形式。全文做缓存,持续一天。
This commit is contained in:
tgly307 2018-08-10 02:27:54 +08:00 committed by DIYgod
parent 9769c550de
commit 1a0ff33603
1 changed files with 41 additions and 10 deletions

View File

@ -1,4 +1,5 @@
const axios = require('../../utils/axios');
const cheerio = require('cheerio');
const config = require('../../config');
module.exports = async (ctx) => {
@ -29,20 +30,50 @@ module.exports = async (ctx) => {
},
});
const data = response.data;
// const data = response.data;
let originalData = [];
if (response.data.d && response.data.d.entrylist) {
originalData = response.data.d && response.data.d.entrylist.slice(0, 5);
}
const resultItems = await Promise.all(
originalData.map(async (item) => {
const resultItem = {
title: item.title,
description: `${(item.content || item.summaryInfo || '无描述').replace(/[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]/g, '')}`,
pubDate: new Date(item.createdAt).toUTCString(),
link: item.originalUrl,
};
if (item.type === 'post') {
const key = 'juejin' + resultItem.link;
const value = await ctx.cache.get(key);
if (value) {
resultItem.description = value;
} else {
const detail = await axios({
method: 'get',
url: item.originalUrl,
headers: {
'User-Agent': config.ua,
Referer: item.originalUrl,
},
});
const content = cheerio.load(detail.data);
resultItem.description = content('.article-content')
.html()
.replace(/[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]/g, '')
.replace(/(<img.*?)(data-src)(.*?>)/g, '$1src$3');
ctx.cache.set(key, resultItem.description, 24 * 60 * 60);
}
}
return Promise.resolve(resultItem);
})
);
ctx.state.data = {
title: `掘金${cat.name}`,
link: `https://juejin.im/welcome/${category}`,
description: `掘金${cat.name}`,
item:
data.d &&
data.d.entrylist &&
data.d.entrylist.map((item) => ({
title: item.title,
description: `${(item.content || item.summaryInfo || '无描述').replace(/[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]/g, '')}`,
pubDate: new Date(item.createdAt).toUTCString(),
link: item.originalUrl,
})),
item: resultItems,
};
};