fix: dockone weekly 404 / get fulltext (#2466)

This commit is contained in:
Cloud 2019-06-24 22:26:46 +08:00 committed by DIYgod
parent 54552fe483
commit be20167a33
1 changed files with 47 additions and 24 deletions

View File

@ -1,42 +1,65 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
function replaceEmpty(str) {
return str.replace(' ', '');
}
const baseUrl = 'http://weekly.dockone.io';
module.exports = async (ctx) => {
const url = `${baseUrl}/issues`;
const baseUrl = 'http://weekly.dockone.io';
const response = await got({
method: 'get',
url: url,
url: baseUrl,
headers: {
Referer: url,
Referer: baseUrl,
},
});
const data = response.data;
const $ = cheerio.load(data);
const list = $('ul.i.issues>li');
const list = $('.aw-common-list .article').get();
const ProcessFeed = (data) => {
const $ = cheerio.load(data);
$('img').each((index, elem) => {
const $elem = $(elem);
$elem.attr('referrerpolicy', 'no-referrer');
});
// 提取内容
return $('.aw-question-detail .content').html();
};
const items = await Promise.all(
list.map(async (item) => {
const $ = cheerio.load(item);
const $a = $('.aw-question-content a');
const link = $a.attr('href');
const cache = await ctx.cache.get(link);
if (cache) {
return Promise.resolve(JSON.parse(cache));
}
const response = await got({
method: 'get',
url: link,
});
const description = ProcessFeed(response.data);
const single = {
title: $a.text(),
description,
link: link,
author: $('.aw-user-name').text(),
};
ctx.cache.set(link, JSON.stringify(single));
return Promise.resolve(single);
})
);
ctx.state.data = {
title: $('title').text(),
link: url,
link: baseUrl,
description: $('meta[name="description"]').attr('content') || $('title').text(),
item:
list &&
list
.map((item, index) => {
item = $(index);
return {
title: `${replaceEmpty(item.find('a>h3>time').text())}[${replaceEmpty(item.find('a>h3>strong').text())}]`,
description: item.find('a>ul>li').text(),
link: `${baseUrl}${item.find('a').attr('href')}`,
};
})
.get(),
item: items,
};
};