diff --git a/docs/en/traditional-media.md b/docs/en/traditional-media.md index fd310fad0..56b661f12 100644 --- a/docs/en/traditional-media.md +++ b/docs/en/traditional-media.md @@ -209,6 +209,16 @@ Provides a better reading experience (full text articles) over the official one. +## The Wall Street Journal (WSJ) + +### News + + + +Provide full article RSS for WSJ topics. + + + ## Yahoo ### News diff --git a/docs/traditional-media.md b/docs/traditional-media.md index 1d39075a6..2f3128a72 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -185,6 +185,16 @@ Solidot 提供的 feed: +## The Wall Street Journal (WSJ) + +### 新闻 + + + +通过提取文章全文,以提供比官方源更佳的阅读体验。 + + + ## UDN ### 轉角國際 diff --git a/lib/router.js b/lib/router.js index 7c70942cf..72025a64d 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3428,4 +3428,7 @@ router.get('/liquipedia/dota2/matches/:id', require('./routes/liquipedia/dota2_m // 哈尔滨市科技局 router.get('/gov/harbin/kjj', require('./routes/gov/harbin/kjj')); +// WSJ +router.get('/wsj/:lang/:category', require('./routes/wsj/index')); + module.exports = router; diff --git a/lib/routes/wsj/index.js b/lib/routes/wsj/index.js new file mode 100644 index 000000000..601e70e90 --- /dev/null +++ b/lib/routes/wsj/index.js @@ -0,0 +1,130 @@ +const parser = require('@/utils/rss-parser'); +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +const categoryToXMLFileName = { + opinion: 'RSSOpinion.xml', + world_news: 'RSSWorldNews.xml', + us_bussiness: 'WSJcomUSBusiness.xml', + market_news: 'RSSMarketsMain.xml', + technology: 'RSSWSJD.xml', + lifestyle: 'RSSLifestyle.xml', +}; + +const categoryToName = { + opinion: 'Opinion', + world_news: 'World News', + us_bussiness: 'U.S. Business', + market_news: 'Markets News', + technology: "Technology: What's News", + lifestyle: 'Lifestyle', +}; + +module.exports = async (ctx) => { + const language = ctx.params.lang; + const category = ctx.params.category; + let rssUrl; + if (language === 'en-us') { + rssUrl = `https://feeds.a.dj.com/rss/${categoryToXMLFileName[category]}`; + } else { + // Doesn't support other languages (e.g. zh-cn, zh-tw, ja) for now + return; + } + const feed = await parser.parseURL(rssUrl); + const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36'; + + const items = await Promise.all( + feed.items.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + // Fetch the AMP version + const url = item.link.replace(/^https:\/\/www\.wsj\.com/, 'https://www.wsj.com/amp'); + const response = await got({ + url, + method: 'get', + headers: { + 'User-Agent': chromeMobileUserAgent, + }, + }); + const html = response.body; + const $ = cheerio.load(html); + const content = $('.articleBody > div[amp-access="access"]'); + + // Cover + const cover = $('.articleLead > div.is-lead-inset > div.header > .img-header > div.image-container > amp-img > img'); + + if (cover.length > 0) { + $(``).insertBefore(content[0].childNodes[0]); + $(cover).remove(); + } + + // Summary + const summary = $('head > meta[name="description"]').attr('content'); + + // Metadata (categories & updatedAt) + const updatedAt = $('meta[itemprop="dateModified"]').attr('content'); + const publishedAt = $('meta[itemprop="datePublished"]').attr('content'); + + const categories = $('meta[name="keywords"]') + .attr('content') + .split(',') + .map((c) => c.trim()); + + // Images + content.find('amp-img').each((i, e) => { + const img = $(`${e.attribs.alt}`); + + // Caption follows, no need to handle caption + $(img).insertBefore(e); + $(e).remove(); + }); + + // iframes (youtube videos and interactive elements) + content.find('amp-iframe').each((i, e) => { + const iframe = $(`