fix: 模型网 (#4264)

This commit is contained in:
cc789461 2020-03-18 10:37:42 +08:00 committed by GitHub
parent 01636ce3b6
commit 1d151ebf6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 9 deletions

View File

@ -11,7 +11,15 @@ module.exports = async (ctx) => {
});
const data = iconv.decode(response.data, 'gb2312');
const $ = cheerio.load(data);
const list = $('ul.text_list.text_list_f14 li').get();
const list = $('ul.text_list.text_list_f14 li')
.get()
.map(function(item) {
const $ = cheerio.load(item);
return baseUrl + $('a').attr('href');
})
.filter(function(link) {
return !link.includes('taobao');
});
const ProcessLink = async (link) => {
const response = await got({
@ -32,26 +40,22 @@ module.exports = async (ctx) => {
desc += next_page.desc;
}
return {
title: $('title').text(),
desc: desc,
};
};
const out = await Promise.all(
list.map(async (item) => {
const $ = cheerio.load(item);
const $a = $('a');
const link = baseUrl + $a.attr('href');
list.map(async (link) => {
const cache = await ctx.cache.get(link);
if (cache) {
return Promise.resolve(JSON.parse(cache));
}
const feed = await ProcessLink(link);
const description = feed.desc;
const single = {
title: $a.text(),
description,
title: feed.title,
description: feed.desc,
link: link,
};
ctx.cache.set(link, JSON.stringify(single));