diff --git a/lib/routes/mediadigest/category.js b/lib/routes/mediadigest/category.js index 757541d4b..aab44b5b8 100644 --- a/lib/routes/mediadigest/category.js +++ b/lib/routes/mediadigest/category.js @@ -1,12 +1,12 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -function getArticle(ctx, list) { +async function getArticle(ctx, list) { // for 每次可爲 rss 値推入 20 篇文章內容 + const res = []; for (let start = 0; start < list.length; start += 20) { // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 - let now = list.slice(start, start+20).forEach((line) => { - ctx.cache.tryGet(line, async() => { + const now = list.slice(start, start + 20).map(async(line) => await ctx.cache.tryGet(line, async() => { const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); const $ = cheerio.load(a_r.data); // title @@ -28,11 +28,11 @@ function getArticle(ctx, list) { pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), link: line, }; - }); - }) + })); // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 - rss.push(...(await Promise.all(now))); + res.push(...await Promise.all(now)); } + return res; } module.exports = async(ctx) => { @@ -40,10 +40,12 @@ module.exports = async(ctx) => { const category = ctx.params.category || 'all'; // for range validation - var num = /^[0-9]+$/; - var current_year = new Date().getFullYear(); + const num = /^[1-9][0-9]{3}$/; + const current_year = new Date().getFullYear(); // placeholder for rss - let rss = []; + let rss = [{ + description: '

Invalid :category? input.

所輸入:category?參數有誤。

所输入:category?参数有误。

' + }]; // cid const cids = [1, 2]; for (let i = 4; i < 28; ++i) { @@ -66,7 +68,7 @@ module.exports = async(ctx) => { return Promise.resolve(list.toArray()); }) ); - const list_all = [...new Set(list_allt.flat())]; // removed duplicates and flatten + const list_all = [...new Set(list_allt.flat())]; // removed duplicates and flatten // 時序排列並抽取最新 50 項 const list = list_all .sort((a, b) => { @@ -77,84 +79,22 @@ module.exports = async(ctx) => { }) .slice(0, 50); // getArticle(ctx, list); - // for 每次會可爲 rss 値段推入 20 篇文章內容 - for (let start = 0; start < list.length; start += 20) { - // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 - let now = list.slice(start, start+20).forEach((line) => { - ctx.cache.tryGet(line, async() => { - const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); - const $ = cheerio.load(a_r.data); - // title - const h1 = $('h1.story-title').text(); - // author - const author_list = $('div.story-author'); - const authors = author_list.map((_index, author) => $(author).text()); - const author = authors.map((_index, author) => `${author}
`); - const s_author = author.toArray().join(''); - const author_block = `

${s_author}

`; - // date - const date = $('div.story-calendar').text(); - // desc - const desc = `${$(author_block)}${$('div.story-content').html()}`; - - return { - title: h1, - description: desc, - pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), - link: line, - }; - }); - }) - // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 - rss.push(...(await Promise.all(now))); - } - } else if (cids.includes(category)) { + rss = await getArticle(ctx, list); + } else if (cids.includes(parseInt(category))) { // 獲取特定 category 文章目錄 const response = await got.get(`https://app3.rthk.hk/mediadigest/category.php?cid=${category}`); const $ = cheerio.load(response.data); const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); - // getArticle(ctx, list); - // for 每次會可爲 rss 値推入 20 篇文章內容 - for (let start = 0; start < list.length; start += 20) { - // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 - let now = list.slice(start, start+20).forEach((line) => { - ctx.cache.tryGet(line, async() => { - const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); - const $ = cheerio.load(a_r.data); - // title - const h1 = $('h1.story-title').text(); - // author - const author_list = $('div.story-author'); - const authors = author_list.map((_index, author) => $(author).text()); - const author = authors.map((_index, author) => `${author}
`); - const s_author = author.toArray().join(''); - const author_block = `

${s_author}

`; - // date - const date = $('div.story-calendar').text(); - // desc - const desc = `${$(author_block)}${$('div.story-content').html()}`; - - return { - title: h1, - description: desc, - pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), - link: line, - }; - }); - }) - // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 - rss.push(...(await Promise.all(now))); - } - } else { - rss = ['

Invalid :category? input.

所輸入:category?參數有誤。

所输入:category?参数有误。

']; + rss = await getArticle(ctx, list.slice(0, 50)); } } // year (specific year range): // if 'all' (all articles in a specific year range); // else 'error'. - else if (range.length === 4 && range !== 0 && num.test(range) && !((range < 1970) || (range > current_year))) { - if (category === 'all') { + else if (num.test(range)) { + const range_num = parseInt(range); + if (category === 'all' && range_num >= 1970 && range_num <= current_year) { // 獲取全站文章 URL const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); const list_all = await Promise.all( @@ -164,71 +104,17 @@ module.exports = async(ctx) => { const $ = cheerio.load(response.data); // 對應文章 URL 與年份 // Ref: https://cythilya.github.io/2016/03/13/jquery-map-grep/ - const list_t = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); + let list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); const year = $('div.category-line div.pull-right').map((_index, date) => new Date($(date).text()).getFullYear()).toArray(); - // 將 list_t 與 year 一一對應,組成 JSON - // Ref: https://stackoverflow.com/questions/30124979/merge-two-arrays-into-one-javascript-object - function article(year, url) { - this.year = year; - this.url = url; - } - var dlist = []; - for (var i = 0; i < list_t.length; i++) { - dlist.push(new article(year[i], list_t[i])); - } + list = list.filter((_url, i) => year[i] === range_num); // 篩出 list (需要抓取的文章 URL 並組成數列) - list = dlist.filter((w) => w.year === range).map((article) => (article.url)); return Promise.resolve(list); }) ); const list = [...new Set(list_all.flat())]; // removed duplicates and flatten - // getArticle(ctx, list); - // for 每次會可爲 rss 値段推入 20 篇文章內容 - for (let start = 0; start < list.length; start += 20) { - // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 - let now = list.slice(start, start+20).forEach((line) => { - ctx.cache.tryGet(line, async() => { - const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); - const $ = cheerio.load(a_r.data); - // title - const h1 = $('h1.story-title').text(); - // author - const author_list = $('div.story-author'); - const authors = author_list.map((_index, author) => $(author).text()); - const author = authors.map((_index, author) => `${author}
`); - const s_author = author.toArray().join(''); - const author_block = `

${s_author}

`; - // date - const date = $('div.story-calendar').text(); - // desc - const desc = `${$(author_block)}${$('div.story-content').html()}`; - - return { - title: h1, - description: desc, - pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), - link: line, - }; - }); - }) - // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 - rss.push(...(await Promise.all(now))); - } - } else { - rss = ['

Please do not filter with both year and category.

請不要同時限定年份及分類。

请不要同时限定年份及分类。

']; - } - } else { - rss = ['

Invalid :range input.

所輸入:range參數有誤。

所输入:range参数有误。

']; - } - -/* // 分批次处理 tasks,每次拉取 20 个 - if (rss.length === 0) { - for (let start = 0; start < tasks.length; start += 20) { - let now = task.slice(start, start+20); - rss.push(...(await Promise.all(now))); + rss = await getArticle(ctx, list); } } - */ ctx.state.data = { title: '傳媒透視',