From d7e758aa0053d405aeea8c4d17868194ee81e813 Mon Sep 17 00:00:00 2001 From: Henry Wang Date: Tue, 28 Jul 2020 16:29:43 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=B7=AF=E9=80=8F=E7=A4=BE=E5=88=86?= =?UTF-8?q?=E7=AB=99=EF=BC=9ACN=20&=20UK=20(#5103)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: proletarius101 --- docs/blog.md | 4 +- docs/en/traditional-media.md | 24 ++++- docs/multimedia.md | 2 - docs/traditional-media.md | 50 +++++++++-- docs/university.md | 12 +-- lib/router.js | 3 + lib/routes/reuters/channel.js | 164 ++++++++++++++++++++++++++++++++++ lib/routes/reuters/utils.js | 66 ++++++++++++++ 8 files changed, 304 insertions(+), 21 deletions(-) create mode 100644 lib/routes/reuters/channel.js create mode 100644 lib/routes/reuters/utils.js diff --git a/docs/blog.md b/docs/blog.md index 8e8d4caf7..34014ec7e 100644 --- a/docs/blog.md +++ b/docs/blog.md @@ -110,9 +110,7 @@ pageClass: routes ### 文章 - - - + ## 王垠博客 diff --git a/docs/en/traditional-media.md b/docs/en/traditional-media.md index cb62e39bf..eaf751b33 100644 --- a/docs/en/traditional-media.md +++ b/docs/en/traditional-media.md @@ -46,7 +46,6 @@ Refer to [Boston.com's feed page](https://www.boston.com/rss-feeds) for tags. Fo Generates full-text that the official feed doesn't provide. Refer to [Chicago Tribune's feed page](https://www.chicagotribune.com/about/ct-chicago-tribune-rss-feeds-htmlstory.html) for categories. For instance, `https://www.chicagotribune.com/arcio/rss/category/nation-world/` corresponds to `/chicagotribune/nation-world`. - ## China Dialogue @@ -89,6 +88,29 @@ Refer to [Chicago Tribune's feed page](https://www.chicagotribune.com/about/ct-c +## Reuters + +### Channel + + + +Supported sub-sites: + +- 中国分站 `cn`： + See [路透社中国分站](../traditional-media.html#lu-tou-she) + +- US site `us`： + | Business | Markets | World | Politics | Tech | Breakingviews | Wealth | Life | + | -------- | ------- | ----- | -------- | ---------- | ------------- | ------ | --------- | + | business | markets | world | politics | technology | breakingviews | wealth | lifestyle | + +- UK site `uk`： + | Business | World | UK | Tech | Money | Breakingviews | Life | + | -------- | ----- | --- | ---------- | --------------- | ------------- | --------- | + | business | world | uk | technology | personalFinance | breakingviews | lifestyle | + + + ## RTHK ### News diff --git a/docs/multimedia.md b/docs/multimedia.md index 382ec7589..f6f122c10 100644 --- a/docs/multimedia.md +++ b/docs/multimedia.md @@ -191,8 +191,6 @@ pageClass: routes - - ## MQube ### 全站最近更新 diff --git a/docs/traditional-media.md b/docs/traditional-media.md index b9fbdadae..6d686949b 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -194,8 +194,6 @@ Category 列表: - - ## 端传媒 ### 端传媒 @@ -306,15 +304,55 @@ category 对应的关键词有 ### 标签 - - - + ## 路透社 ### 实时资讯 - + + +### 频道 + + + +支持语言列表 + +- 中国分站 `cn`： + + - 主频道: + + | 深度分析 | 时事要闻 | 生活 | 投资 | + | -------- | ----------- | ---- | --------- | + | analyses | generalnews | life | investing | + + - 资讯子频道: + + | 中国财经 | 国际财经 | 新闻人物 | 财经视点 | + | -------- | --------------------- | --------- | -------- | + | china | internationalbusiness | newsmaker | opinions | + + - 专栏子频道: + + | 中国财经专栏 | 国际财经专栏 | 大宗商品专栏 | + | ------------ | ------------ | ------------ | + | CnColumn | IntColumn | ComColumn | + +- 美国分站 `us`： + + - 主频道: + + | Business | Markets | World | Politics | Tech | Breakingviews | Wealth | Life | + | -------- | ------- | ----- | -------- | ---------- | ------------- | ------ | --------- | + | business | markets | world | politics | technology | breakingviews | wealth | lifestyle | + +- 英国分站 `uk`： + + - 主频道: + + | Business | Markets | World | UK | Tech | Money | Breakingviews | Sport | Life | + | -------- | ------- | ----- | -- | ---------- | --------------- | ------------- | ------ | --------- | + | business | markets | world | uk | technology | personalFinance | breakingviews | sports | lifestyle | diff --git a/docs/university.md b/docs/university.md index 904e2ce06..5f3b7c9ac 100644 --- a/docs/university.md +++ b/docs/university.md @@ -1358,15 +1358,11 @@ type 列表： ### 信息与工程学院动态公告列表 - - - + ### 信息与工程学院公告详情 - - - + ### 教务处 @@ -1380,9 +1376,7 @@ type 列表： ### 教务处通告详情 - - - + ## 天津大学 diff --git a/lib/router.js b/lib/router.js index 0355e5e22..6f451d90e 100644 --- a/lib/router.js +++ b/lib/router.js @@ -2062,6 +2062,9 @@ router.get('/netease/dy/:id', require('./routes/netease/dy')); // 海猫吧 router.get('/haimaoba/:id?', require('./routes/haimaoba/comics')); +// 路透社 +router.get('/reuters/channel/:site/:channel', require('./routes/reuters/channel')); + // 蒲公英 router.get('/pgyer/:app?', require('./routes/pgyer/app')); diff --git a/lib/routes/reuters/channel.js b/lib/routes/reuters/channel.js new file mode 100644 index 000000000..9d02a5e54 --- /dev/null +++ b/lib/routes/reuters/channel.js @@ -0,0 +1,164 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const utils = require('./utils'); + +module.exports = async (ctx) => { + const { site } = ctx.params || 'cn'; + const homePage = `https://${site === 'us' ? 'www' : site}.reuters.com`; + + let title = 'Reuters ', + link = `https://${site === 'us' ? 'www' : site}.reuters.com/news/`, + linkSelector; + + const { channel } = ctx.params; + + if (channel) { + if (site === 'cn') { + title = '路透社 '; + linkSelector = '.inlineLinks a'; + switch (channel) { + case 'china': + title += '中国财经'; + break; + case 'internationalbusiness': + title += '国际财经'; + break; + case 'newsmaker': + title += '新闻人物'; + break; + case 'opinions': + title += '财经视点'; + break; + case 'analyses': + title += '深度分析'; + break; + case 'generalnews': + title += '时事要闻'; + break; + case 'CnColumn': + title += '中国财经专栏'; + break; + case 'ComColumn': + title += '大宗商品专栏'; + break; + case 'IntColumn': + title += '国际财经专栏'; + break; + case 'investing': + title += '投资'; + link = homePage; + break; + case 'life': + title += '生活'; + link = homePage; + break; + default: + break; + } + } else if (site === 'uk') { + title += 'UK '; + linkSelector = '.story-content a'; + switch (channel) { + case 'business': + title += 'Business'; + link = homePage; + break; + case 'markets': + linkSelector = '.moduleBody a, .story-content a'; + title += 'Markets'; + link = 'https://uk.reuters.com/business/'; + break; + case 'world': + title += 'World'; + break; + case 'uk': + title += 'UK'; + break; + case 'technology': + title += 'Tech'; + break; + case 'personalFinance': + title += 'Money'; + link = 'https://uk.reuters.com/business'; + break; + case 'breakingviews': + title += 'Breakingviews'; + link = homePage; + break; + case 'sports': + linkSelector = '.story-content a, .story-title a'; + title += 'Sport'; + break; + case 'lifestyle': + title += 'Life'; + break; + default: + break; + } + } else if (site === 'us') { + linkSelector = '.story-content a,'; + switch (channel) { + case 'business': + title += 'Business'; + link = homePage; + break; + case 'markets': + linkSelector = '.moduleBody a, .story-content a'; + title += 'Markets'; + link = 'https://www.reuters.com/finance/'; + break; + case 'world': + title += 'World'; + break; + case 'politics': + title += 'Politics'; + link = 'https://www.reuters.com/'; + break; + case 'technology': + title += 'Tech'; + break; + case 'breakingviews': + title += 'Breakingviews'; + link = homePage; + break; + case 'wealth': + linkSelector = '.story-content a, .story-title a'; + title += 'Wealth'; + link = 'https://www.reuters.com/finance/'; + break; + case 'lifestyle': + title += 'Life'; + break; + default: + break; + } + } + link += channel; + } + + const response = await got.get(link); + + const $ = cheerio.load(response.data); + + const links = $(linkSelector).map((i, e) => homePage + e.attribs.href); + + const items = await Promise.all( + links.splice(0, 10).map(async (link) => { + const cache = await ctx.cache.get(link); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + const single = await utils.ProcessFeed(link); + ctx.cache.set(link, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + + ctx.state.data = { + title, + link, + description: title, + item: items, + }; +}; diff --git a/lib/routes/reuters/utils.js b/lib/routes/reuters/utils.js new file mode 100644 index 000000000..a96034f62 --- /dev/null +++ b/lib/routes/reuters/utils.js @@ -0,0 +1,66 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +const ProcessFeed = async (link) => { + const response = await got.get(link); + const $ = cheerio.load(response.data); + + const pubDate = $('meta[property="og:article:published_time"]')[0].attribs.content; + const title = $('meta[property="og:title"]')[0].attribs.content; + const author = $('meta[property="og:article:author"]')[0].attribs.content; + const cover = $('meta[property="og:image"]')[0].attribs.content; + const description = $('.StandardArticleBody_body'); + + // if the article's cover photo is not the meaningless logo + if (cover !== 'https://s4.reutersmedia.net/resources_v2/images/rcom-default.png') { + const image = $('.PrimaryAsset_container img'); + + if (image.length > 0) { + image[0].attribs.src = cover; + } + } + + // handle slideshows and videos + const pageDataPattern = new RegExp(`(?<=)`); + const pageData = JSON.parse(response.data.match(pageDataPattern)[0]); + + // keys of this json has random tails, so we have to iterate through them + Object.keys(pageData).forEach((key) => { + // videos and slideshow appear only in `article_list` + if (!key.startsWith('article_list')) { + return; + } + + // add full-res pictures at the end + const slideshowData = pageData[key].first_article.images; + slideshowData && + slideshowData.forEach((imgData) => { + description.insertAfter(`
+ ${imgData.title} +
${imgData.caption}
+
`); + }); + + // add full-res pictures at the beginning + const videoData = pageData[key].first_article.videos; + videoData && + videoData.forEach((videoData) => { + description.insertBefore(` + ${videoData.caption}`); + }); + }); + + // remove useless DOMs + description.find('.Image_expand-button, .LazyImage_fallback, .Slideshow_container, .Slideshow_caption, .Slideshow_expand-button, .Attribution_container, .StandardArticleBody_trustBadgeContainer').remove(); + + return { link, author, pubDate, title, description: description.html() }; +}; + +module.exports = { + ProcessFeed, +};