diff --git a/lib/router.js b/lib/router.js index dcb959fb8..488de8eb2 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3767,6 +3767,6 @@ router.get('/sciencenet/blog/:type?/:time?/:sort?', require('./routes/sciencenet router.get('/dailyart/:language?', require('./routes/dailyart/index')); // Media Digest -router.get('/mediadigest/:category?', require('./routes/mediadigest/category')); +router.get('/mediadigest/:range/:category?', require('./routes/mediadigest/category')); module.exports = router; diff --git a/lib/routes/mediadigest/category.js b/lib/routes/mediadigest/category.js index b42a27f64..757541d4b 100644 --- a/lib/routes/mediadigest/category.js +++ b/lib/routes/mediadigest/category.js @@ -1,79 +1,238 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -function getArticle(ctx, line, tasks) { - const article = ctx.cache.tryGet(line, async () => { - const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); - const $ = cheerio.load(a_r.data); - // title - const h1 = $('h1.story-title').text(); - // author - const author_list = $('div.story-author'); - const authors = author_list.map((_index, author) => $(author).text()); - const author = authors.map((_index, author) => `${author}
`); - const s_author = author.toArray().join(''); - const author_block = `

${s_author}

`; - // date - const date = $('div.story-calendar').text(); - // desc - const desc = `${$(author_block)}${$('div.story-content').html()}`; +function getArticle(ctx, list) { + // for 每次可爲 rss 値推入 20 篇文章內容 + for (let start = 0; start < list.length; start += 20) { + // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 + let now = list.slice(start, start+20).forEach((line) => { + ctx.cache.tryGet(line, async() => { + const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); + const $ = cheerio.load(a_r.data); + // title + const h1 = $('h1.story-title').text(); + // author + const author_list = $('div.story-author'); + const authors = author_list.map((_index, author) => $(author).text()); + const author = authors.map((_index, author) => `${author}
`); + const s_author = author.toArray().join(''); + const author_block = `

${s_author}

`; + // date + const date = $('div.story-calendar').text(); + // desc + const desc = `${$(author_block)}${$('div.story-content').html()}`; - return { - title: h1, - description: desc, - pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), - link: line, - }; - }); - tasks.push(article); + return { + title: h1, + description: desc, + pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), + link: line, + }; + }); + }) + // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 + rss.push(...(await Promise.all(now))); + } } -module.exports = async (ctx) => { +module.exports = async(ctx) => { + const range = ctx.params.range || 'latest'; const category = ctx.params.category || 'all'; - const tasks = []; - if (category !== 'all') { - const response = await got.get(`https://app3.rthk.hk/mediadigest/category.php?cid=${category}`); - const $ = cheerio.load(response.data); - const list = $('div.category-line'); - - list.map((_index, line) => $(line).find('a').first().attr('href')).each((_index, line) => { - getArticle(ctx, line, tasks); - }); - } else { - const cids = [1, 2]; - for (let i = 4; i < 28; ++i) { - cids.push(i); - } - const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); - const list_allt = await Promise.all( - urls.map(async (url) => { - const response = await got.get(url); - const $ = cheerio.load(response.data); - const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')); - return Promise.resolve(list.toArray()); - }) - ); - const list_all = [...new Set(list_allt.flat())]; - - const list_recent_20 = list_all - .sort((a, b) => { - const aid_a = a.match(/aid=(\d+)/)[1]; - const aid_b = b.match(/aid=(\d+)/)[1]; - // reverse - return aid_b - aid_a; - }) - .slice(0, 20); - list_recent_20.forEach((line) => { - getArticle(ctx, line, tasks); - }); + // for range validation + var num = /^[0-9]+$/; + var current_year = new Date().getFullYear(); + // placeholder for rss + let rss = []; + // cid + const cids = [1, 2]; + for (let i = 4; i < 28; ++i) { + cids.push(i); } - const rss = await Promise.all(tasks); + // latest (50 articles): + // if 'all' (latest 50 articles of the site); + // else if 'cid' (all articles of a specific category); + // else 'error'. + if (range === 'latest') { + if (category === 'all') { + // 獲取全站文章 URL + const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); + const list_allt = await Promise.all( + urls.map(async(url) => { + const response = await got.get(url); + const $ = cheerio.load(response.data); + const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')); + return Promise.resolve(list.toArray()); + }) + ); + const list_all = [...new Set(list_allt.flat())]; // removed duplicates and flatten + // 時序排列並抽取最新 50 項 + const list = list_all + .sort((a, b) => { + const aid_a = a.match(/aid=(\d+)/)[1]; + const aid_b = b.match(/aid=(\d+)/)[1]; + // reverse + return aid_b - aid_a; + }) + .slice(0, 50); + // getArticle(ctx, list); + // for 每次會可爲 rss 値段推入 20 篇文章內容 + for (let start = 0; start < list.length; start += 20) { + // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 + let now = list.slice(start, start+20).forEach((line) => { + ctx.cache.tryGet(line, async() => { + const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); + const $ = cheerio.load(a_r.data); + // title + const h1 = $('h1.story-title').text(); + // author + const author_list = $('div.story-author'); + const authors = author_list.map((_index, author) => $(author).text()); + const author = authors.map((_index, author) => `${author}
`); + const s_author = author.toArray().join(''); + const author_block = `

${s_author}

`; + // date + const date = $('div.story-calendar').text(); + // desc + const desc = `${$(author_block)}${$('div.story-content').html()}`; + + return { + title: h1, + description: desc, + pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), + link: line, + }; + }); + }) + // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 + rss.push(...(await Promise.all(now))); + } + } else if (cids.includes(category)) { + // 獲取特定 category 文章目錄 + const response = await got.get(`https://app3.rthk.hk/mediadigest/category.php?cid=${category}`); + const $ = cheerio.load(response.data); + + const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); + // getArticle(ctx, list); + // for 每次會可爲 rss 値推入 20 篇文章內容 + for (let start = 0; start < list.length; start += 20) { + // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 + let now = list.slice(start, start+20).forEach((line) => { + ctx.cache.tryGet(line, async() => { + const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); + const $ = cheerio.load(a_r.data); + // title + const h1 = $('h1.story-title').text(); + // author + const author_list = $('div.story-author'); + const authors = author_list.map((_index, author) => $(author).text()); + const author = authors.map((_index, author) => `${author}
`); + const s_author = author.toArray().join(''); + const author_block = `

${s_author}

`; + // date + const date = $('div.story-calendar').text(); + // desc + const desc = `${$(author_block)}${$('div.story-content').html()}`; + + return { + title: h1, + description: desc, + pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), + link: line, + }; + }); + }) + // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 + rss.push(...(await Promise.all(now))); + } + } else { + rss = ['

Invalid :category? input.

所輸入:category?參數有誤。

所输入:category?参数有误。

']; + } + } + // year (specific year range): + // if 'all' (all articles in a specific year range); + // else 'error'. + else if (range.length === 4 && range !== 0 && num.test(range) && !((range < 1970) || (range > current_year))) { + if (category === 'all') { + // 獲取全站文章 URL + const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); + const list_all = await Promise.all( + // 每個任務是篩選出一個文章目錄裏需要抓取的文章 URL,以供後續「獲取全文」 + urls.map(async(url) => { + const response = await got.get(url); + const $ = cheerio.load(response.data); + // 對應文章 URL 與年份 + // Ref: https://cythilya.github.io/2016/03/13/jquery-map-grep/ + const list_t = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); + const year = $('div.category-line div.pull-right').map((_index, date) => new Date($(date).text()).getFullYear()).toArray(); + // 將 list_t 與 year 一一對應,組成 JSON + // Ref: https://stackoverflow.com/questions/30124979/merge-two-arrays-into-one-javascript-object + function article(year, url) { + this.year = year; + this.url = url; + } + var dlist = []; + for (var i = 0; i < list_t.length; i++) { + dlist.push(new article(year[i], list_t[i])); + } + // 篩出 list (需要抓取的文章 URL 並組成數列) + list = dlist.filter((w) => w.year === range).map((article) => (article.url)); + return Promise.resolve(list); + }) + ); + const list = [...new Set(list_all.flat())]; // removed duplicates and flatten + // getArticle(ctx, list); + // for 每次會可爲 rss 値段推入 20 篇文章內容 + for (let start = 0; start < list.length; start += 20) { + // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 + let now = list.slice(start, start+20).forEach((line) => { + ctx.cache.tryGet(line, async() => { + const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); + const $ = cheerio.load(a_r.data); + // title + const h1 = $('h1.story-title').text(); + // author + const author_list = $('div.story-author'); + const authors = author_list.map((_index, author) => $(author).text()); + const author = authors.map((_index, author) => `${author}
`); + const s_author = author.toArray().join(''); + const author_block = `

${s_author}

`; + // date + const date = $('div.story-calendar').text(); + // desc + const desc = `${$(author_block)}${$('div.story-content').html()}`; + + return { + title: h1, + description: desc, + pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), + link: line, + }; + }); + }) + // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 + rss.push(...(await Promise.all(now))); + } + } else { + rss = ['

Please do not filter with both year and category.

請不要同時限定年份及分類。

请不要同时限定年份及分类。

']; + } + } else { + rss = ['

Invalid :range input.

所輸入:range參數有誤。

所输入:range参数有误。

']; + } + +/* // 分批次处理 tasks,每次拉取 20 个 + if (rss.length === 0) { + for (let start = 0; start < tasks.length; start += 20) { + let now = task.slice(start, start+20); + rss.push(...(await Promise.all(now))); + } + } + */ ctx.state.data = { title: '傳媒透視', link: 'https://app3.rthk.hk/mediadigest/index.php', item: rss, }; -}; +}; \ No newline at end of file