fix
This commit is contained in:
@@ -1,12 +1,12 @@
|
||||
const got = require('@/utils/got');
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
function getArticle(ctx, list) {
|
||||
async function getArticle(ctx, list) {
|
||||
// for 每次可爲 rss 値推入 20 篇文章內容
|
||||
const res = [];
|
||||
for (let start = 0; start < list.length; start += 20) {
|
||||
// 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務
|
||||
let now = list.slice(start, start+20).forEach((line) => {
|
||||
ctx.cache.tryGet(line, async() => {
|
||||
const now = list.slice(start, start + 20).map(async(line) => await ctx.cache.tryGet(line, async() => {
|
||||
const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`);
|
||||
const $ = cheerio.load(a_r.data);
|
||||
// title
|
||||
@@ -28,11 +28,11 @@ function getArticle(ctx, list) {
|
||||
pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(),
|
||||
link: line,
|
||||
};
|
||||
});
|
||||
})
|
||||
}));
|
||||
// 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値
|
||||
rss.push(...(await Promise.all(now)));
|
||||
res.push(...await Promise.all(now));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
module.exports = async(ctx) => {
|
||||
@@ -40,10 +40,12 @@ module.exports = async(ctx) => {
|
||||
const category = ctx.params.category || 'all';
|
||||
|
||||
// for range validation
|
||||
var num = /^[0-9]+$/;
|
||||
var current_year = new Date().getFullYear();
|
||||
const num = /^[1-9][0-9]{3}$/;
|
||||
const current_year = new Date().getFullYear();
|
||||
// placeholder for rss
|
||||
let rss = [];
|
||||
let rss = [{
|
||||
description: '<p>Invalid <code>:category?</code> input.</p><p>所輸入<code>:category?</code>參數有誤。</p><p>所输入<code>:category?</code>参数有误。</p>'
|
||||
}];
|
||||
// cid
|
||||
const cids = [1, 2];
|
||||
for (let i = 4; i < 28; ++i) {
|
||||
@@ -66,7 +68,7 @@ module.exports = async(ctx) => {
|
||||
return Promise.resolve(list.toArray());
|
||||
})
|
||||
);
|
||||
const list_all = [...new Set(list_allt.flat())]; // removed duplicates and flatten
|
||||
const list_all = [...new Set(list_allt.flat())]; // removed duplicates and flatten
|
||||
// 時序排列並抽取最新 50 項
|
||||
const list = list_all
|
||||
.sort((a, b) => {
|
||||
@@ -77,84 +79,22 @@ module.exports = async(ctx) => {
|
||||
})
|
||||
.slice(0, 50);
|
||||
// getArticle(ctx, list);
|
||||
// for 每次會可爲 rss 値段推入 20 篇文章內容
|
||||
for (let start = 0; start < list.length; start += 20) {
|
||||
// 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務
|
||||
let now = list.slice(start, start+20).forEach((line) => {
|
||||
ctx.cache.tryGet(line, async() => {
|
||||
const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`);
|
||||
const $ = cheerio.load(a_r.data);
|
||||
// title
|
||||
const h1 = $('h1.story-title').text();
|
||||
// author
|
||||
const author_list = $('div.story-author');
|
||||
const authors = author_list.map((_index, author) => $(author).text());
|
||||
const author = authors.map((_index, author) => `<b>${author}</b><br>`);
|
||||
const s_author = author.toArray().join('');
|
||||
const author_block = `<blockquote><p>${s_author}</p></blockquote>`;
|
||||
// date
|
||||
const date = $('div.story-calendar').text();
|
||||
// desc
|
||||
const desc = `${$(author_block)}${$('div.story-content').html()}`;
|
||||
|
||||
return {
|
||||
title: h1,
|
||||
description: desc,
|
||||
pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(),
|
||||
link: line,
|
||||
};
|
||||
});
|
||||
})
|
||||
// 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値
|
||||
rss.push(...(await Promise.all(now)));
|
||||
}
|
||||
} else if (cids.includes(category)) {
|
||||
rss = await getArticle(ctx, list);
|
||||
} else if (cids.includes(parseInt(category))) {
|
||||
// 獲取特定 category 文章目錄
|
||||
const response = await got.get(`https://app3.rthk.hk/mediadigest/category.php?cid=${category}`);
|
||||
const $ = cheerio.load(response.data);
|
||||
|
||||
const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray();
|
||||
// getArticle(ctx, list);
|
||||
// for 每次會可爲 rss 値推入 20 篇文章內容
|
||||
for (let start = 0; start < list.length; start += 20) {
|
||||
// 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務
|
||||
let now = list.slice(start, start+20).forEach((line) => {
|
||||
ctx.cache.tryGet(line, async() => {
|
||||
const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`);
|
||||
const $ = cheerio.load(a_r.data);
|
||||
// title
|
||||
const h1 = $('h1.story-title').text();
|
||||
// author
|
||||
const author_list = $('div.story-author');
|
||||
const authors = author_list.map((_index, author) => $(author).text());
|
||||
const author = authors.map((_index, author) => `<b>${author}</b><br>`);
|
||||
const s_author = author.toArray().join('');
|
||||
const author_block = `<blockquote><p>${s_author}</p></blockquote>`;
|
||||
// date
|
||||
const date = $('div.story-calendar').text();
|
||||
// desc
|
||||
const desc = `${$(author_block)}${$('div.story-content').html()}`;
|
||||
|
||||
return {
|
||||
title: h1,
|
||||
description: desc,
|
||||
pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(),
|
||||
link: line,
|
||||
};
|
||||
});
|
||||
})
|
||||
// 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値
|
||||
rss.push(...(await Promise.all(now)));
|
||||
}
|
||||
} else {
|
||||
rss = ['<p>Invalid <code>:category?</code> input.</p><p>所輸入<code>:category?</code>參數有誤。</p><p>所输入<code>:category?</code>参数有误。</p>'];
|
||||
rss = await getArticle(ctx, list.slice(0, 50));
|
||||
}
|
||||
}
|
||||
// year (specific year range):
|
||||
// if 'all' (all articles in a specific year range);
|
||||
// else 'error'.
|
||||
else if (range.length === 4 && range !== 0 && num.test(range) && !((range < 1970) || (range > current_year))) {
|
||||
if (category === 'all') {
|
||||
else if (num.test(range)) {
|
||||
const range_num = parseInt(range);
|
||||
if (category === 'all' && range_num >= 1970 && range_num <= current_year) {
|
||||
// 獲取全站文章 URL
|
||||
const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`);
|
||||
const list_all = await Promise.all(
|
||||
@@ -164,71 +104,17 @@ module.exports = async(ctx) => {
|
||||
const $ = cheerio.load(response.data);
|
||||
// 對應文章 URL 與年份
|
||||
// Ref: https://cythilya.github.io/2016/03/13/jquery-map-grep/
|
||||
const list_t = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray();
|
||||
let list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray();
|
||||
const year = $('div.category-line div.pull-right').map((_index, date) => new Date($(date).text()).getFullYear()).toArray();
|
||||
// 將 list_t 與 year 一一對應,組成 JSON
|
||||
// Ref: https://stackoverflow.com/questions/30124979/merge-two-arrays-into-one-javascript-object
|
||||
function article(year, url) {
|
||||
this.year = year;
|
||||
this.url = url;
|
||||
}
|
||||
var dlist = [];
|
||||
for (var i = 0; i < list_t.length; i++) {
|
||||
dlist.push(new article(year[i], list_t[i]));
|
||||
}
|
||||
list = list.filter((_url, i) => year[i] === range_num);
|
||||
// 篩出 list (需要抓取的文章 URL 並組成數列)
|
||||
list = dlist.filter((w) => w.year === range).map((article) => (article.url));
|
||||
return Promise.resolve(list);
|
||||
})
|
||||
);
|
||||
const list = [...new Set(list_all.flat())]; // removed duplicates and flatten
|
||||
// getArticle(ctx, list);
|
||||
// for 每次會可爲 rss 値段推入 20 篇文章內容
|
||||
for (let start = 0; start < list.length; start += 20) {
|
||||
// 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務
|
||||
let now = list.slice(start, start+20).forEach((line) => {
|
||||
ctx.cache.tryGet(line, async() => {
|
||||
const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`);
|
||||
const $ = cheerio.load(a_r.data);
|
||||
// title
|
||||
const h1 = $('h1.story-title').text();
|
||||
// author
|
||||
const author_list = $('div.story-author');
|
||||
const authors = author_list.map((_index, author) => $(author).text());
|
||||
const author = authors.map((_index, author) => `<b>${author}</b><br>`);
|
||||
const s_author = author.toArray().join('');
|
||||
const author_block = `<blockquote><p>${s_author}</p></blockquote>`;
|
||||
// date
|
||||
const date = $('div.story-calendar').text();
|
||||
// desc
|
||||
const desc = `${$(author_block)}${$('div.story-content').html()}`;
|
||||
|
||||
return {
|
||||
title: h1,
|
||||
description: desc,
|
||||
pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(),
|
||||
link: line,
|
||||
};
|
||||
});
|
||||
})
|
||||
// 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値
|
||||
rss.push(...(await Promise.all(now)));
|
||||
}
|
||||
} else {
|
||||
rss = ['<p>Please do not filter with both year and category.</p><p>請不要同時限定年份及分類。</p><p>请不要同时限定年份及分类。</p>'];
|
||||
}
|
||||
} else {
|
||||
rss = ['<p>Invalid <code>:range</code> input.</p><p>所輸入<code>:range</code>參數有誤。</p><p>所输入<code>:range</code>参数有误。</p>'];
|
||||
}
|
||||
|
||||
/* // 分批次处理 tasks,每次拉取 20 个
|
||||
if (rss.length === 0) {
|
||||
for (let start = 0; start < tasks.length; start += 20) {
|
||||
let now = task.slice(start, start+20);
|
||||
rss.push(...(await Promise.all(now)));
|
||||
rss = await getArticle(ctx, list);
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
ctx.state.data = {
|
||||
title: '傳媒透視',
|
||||
|
||||
Reference in New Issue
Block a user