xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

just async

This commit is contained in:
ギャラ 2020-12-29 09:34:39 +00:00 committed by GitHub
parent d0130d9dcd
commit 473cc49d02
1 changed files with 26 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
lib/routes/mediadigest/category.js
View File

@ -2,11 +2,9 @@ const got = require('@/utils/got');
const cheerio = require('cheerio');
async function getArticle(ctx, list) {
// for 每次可爲 rss 値推入 20 篇文章內容
const res = [];
for (let start = 0; start < list.length; start += 20) {
// 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務
const now = list.slice(start, start + 20).map(async(line) => await ctx.cache.tryGet(line, async() => {
const now = list.map(
async (line) =>
await ctx.cache.tryGet(line, async () => {
const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`);
const $ = cheerio.load(a_r.data);
// title
@ -28,14 +26,13 @@ async function getArticle(ctx, list) {
pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(),
link: line,
};
}));
// 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値
res.push(...await Promise.all(now));
}
return res;
})
);
// 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値
return await Promise.all(now);
}
module.exports = async(ctx) => {
module.exports = async (ctx) => {
const range = ctx.params.range || 'latest';
const category = ctx.params.category || 'all';
@ -43,9 +40,11 @@ module.exports = async(ctx) => {
const num = /^[1-9][0-9]{3}$/;
const current_year = new Date().getFullYear();
// placeholder for rss
let rss = [{
description: '<p>Invalid <code>:category?</code> input.</p><p>所輸入<code>:category?</code>參數有誤。</p><p>所输入<code>:category?</code>参数有误。</p>'
}];
let rss = [
{
description: '<p>Invalid <code>:category?</code> input.</p><p>所輸入<code>:category?</code>參數有誤。</p><p>所输入<code>:category?</code>参数有误。</p>',
},
];
// cid
const cids = [1, 2];
for (let i = 4; i < 28; ++i) {
@ -61,7 +60,7 @@ module.exports = async(ctx) => {
// 獲取全站文章 URL
const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`);
const list_allt = await Promise.all(
urls.map(async(url) => {
urls.map(async (url) => {
const response = await got.get(url);
const $ = cheerio.load(response.data);
const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href'));
@ -85,7 +84,9 @@ module.exports = async(ctx) => {
const response = await got.get(`https://app3.rthk.hk/mediadigest/category.php?cid=${category}`);
const $ = cheerio.load(response.data);
const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray();
const list = $('div.category-line')
.map((_index, line) => $(line).find('a').first().attr('href'))
.toArray();
rss = await getArticle(ctx, list.slice(0, 50));
}
}
@ -99,20 +100,24 @@ module.exports = async(ctx) => {
const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`);
const list_all = await Promise.all(
// 每個任務是篩選出一個文章目錄裏需要抓取的文章 URL，以供後續「獲取全文」
urls.map(async(url) => {
urls.map(async (url) => {
const response = await got.get(url);
const $ = cheerio.load(response.data);
// 對應文章 URL 與年份
// Ref: https://cythilya.github.io/2016/03/13/jquery-map-grep/
let list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray();
const year = $('div.category-line div.pull-right').map((_index, date) => new Date($(date).text()).getFullYear()).toArray();
let list = $('div.category-line')
.map((_index, line) => $(line).find('a').first().attr('href'))
.toArray();
const year = $('div.category-line div.pull-right')
.map((_index, date) => new Date($(date).text()).getFullYear())
.toArray();
list = list.filter((_url, i) => year[i] === range_num);
// 篩出 list (需要抓取的文章 URL 並組成數列)
return Promise.resolve(list);
})
);
const list = [...new Set(list_all.flat())]; // removed duplicates and flatten
rss = await getArticle(ctx, list);
rss = await getArticle(ctx, list.slice(0, 50));
}
}
@ -121,4 +126,4 @@ module.exports = async(ctx) => {
link: 'https://app3.rthk.hk/mediadigest/index.php',
item: rss,
};
};
};