feat: add wsj opinion (#6057)
This commit is contained in:
oppilate
GitHub
parent
3a0d241801
commit
6de46e59cb
|
|
@ -209,6 +209,16 @@ Provides a better reading experience (full text articles) over the official one.
|
|||
|
||||
</RouteEn>
|
||||
|
||||
## The Wall Street Journal (WSJ)
|
||||
|
||||
### News
|
||||
|
||||
<Route author="oppilate" example="/wsj/en-us/opinion" path="/wsj/:lang/:category" :paramsDesc="['Language, `en-us` only for now', 'Category, see [RSS feeds in WSJ.com](https://www.wsj.com/news/rss-news-and-feeds)']">
|
||||
|
||||
Provide full article RSS for WSJ topics.
|
||||
|
||||
</Route>
|
||||
|
||||
## Yahoo
|
||||
|
||||
### News
|
||||
|
|
|
|||
|
|
@ -185,6 +185,16 @@ Solidot 提供的 feed:
|
|||
|
||||
</Route>
|
||||
|
||||
## The Wall Street Journal (WSJ)
|
||||
|
||||
### 新闻
|
||||
|
||||
<Route author="oppilate" example="/wsj/en-us/opinion" path="/wsj/:lang/:category" :paramsDesc="['语言，暂时仅限 `en-us`', '分类，见[WSJ 官网 RSS 分类](https://www.wsj.com/news/rss-news-and-feeds)']">
|
||||
|
||||
通过提取文章全文，以提供比官方源更佳的阅读体验。
|
||||
|
||||
</Route>
|
||||
|
||||
## UDN
|
||||
|
||||
### 轉角國際
|
||||
|
|
|
|||
|
|
@ -3428,4 +3428,7 @@ router.get('/liquipedia/dota2/matches/:id', require('./routes/liquipedia/dota2_m
|
|||
// 哈尔滨市科技局
|
||||
router.get('/gov/harbin/kjj', require('./routes/gov/harbin/kjj'));
|
||||
|
||||
// WSJ
|
||||
router.get('/wsj/:lang/:category', require('./routes/wsj/index'));
|
||||
|
||||
module.exports = router;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
const parser = require('@/utils/rss-parser');
|
||||
const cheerio = require('cheerio');
|
||||
const got = require('@/utils/got');
|
||||
|
||||
const categoryToXMLFileName = {
|
||||
opinion: 'RSSOpinion.xml',
|
||||
world_news: 'RSSWorldNews.xml',
|
||||
us_bussiness: 'WSJcomUSBusiness.xml',
|
||||
market_news: 'RSSMarketsMain.xml',
|
||||
technology: 'RSSWSJD.xml',
|
||||
lifestyle: 'RSSLifestyle.xml',
|
||||
};
|
||||
|
||||
const categoryToName = {
|
||||
opinion: 'Opinion',
|
||||
world_news: 'World News',
|
||||
us_bussiness: 'U.S. Business',
|
||||
market_news: 'Markets News',
|
||||
technology: "Technology: What's News",
|
||||
lifestyle: 'Lifestyle',
|
||||
};
|
||||
|
||||
module.exports = async (ctx) => {
|
||||
const language = ctx.params.lang;
|
||||
const category = ctx.params.category;
|
||||
let rssUrl;
|
||||
if (language === 'en-us') {
|
||||
rssUrl = `https://feeds.a.dj.com/rss/${categoryToXMLFileName[category]}`;
|
||||
} else {
|
||||
// Doesn't support other languages (e.g. zh-cn, zh-tw, ja) for now
|
||||
return;
|
||||
}
|
||||
const feed = await parser.parseURL(rssUrl);
|
||||
const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
|
||||
|
||||
const items = await Promise.all(
|
||||
feed.items.map(
|
||||
async (item) =>
|
||||
await ctx.cache.tryGet(item.link, async () => {
|
||||
// Fetch the AMP version
|
||||
const url = item.link.replace(/^https:\/\/www\.wsj\.com/, 'https://www.wsj.com/amp');
|
||||
const response = await got({
|
||||
url,
|
||||
method: 'get',
|
||||
headers: {
|
||||
'User-Agent': chromeMobileUserAgent,
|
||||
},
|
||||
});
|
||||
const html = response.body;
|
||||
const $ = cheerio.load(html);
|
||||
const content = $('.articleBody > div[amp-access="access"]');
|
||||
|
||||
// Cover
|
||||
const cover = $('.articleLead > div.is-lead-inset > div.header > .img-header > div.image-container > amp-img > img');
|
||||
|
||||
if (cover.length > 0) {
|
||||
$(`<img src=${cover[0].attribs.content}>`).insertBefore(content[0].childNodes[0]);
|
||||
$(cover).remove();
|
||||
}
|
||||
|
||||
// Summary
|
||||
const summary = $('head > meta[name="description"]').attr('content');
|
||||
|
||||
// Metadata (categories & updatedAt)
|
||||
const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
|
||||
const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
|
||||
|
||||
const categories = $('meta[name="keywords"]')
|
||||
.attr('content')
|
||||
.split(',')
|
||||
.map((c) => c.trim());
|
||||
|
||||
// Images
|
||||
content.find('amp-img').each((i, e) => {
|
||||
const img = $(`<img width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}" alt="${e.attribs.alt}">`);
|
||||
|
||||
// Caption follows, no need to handle caption
|
||||
$(img).insertBefore(e);
|
||||
$(e).remove();
|
||||
});
|
||||
|
||||
// iframes (youtube videos and interactive elements)
|
||||
content.find('amp-iframe').each((i, e) => {
|
||||
const iframe = $(`<iframe width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}">`);
|
||||
$(iframe).insertBefore(e);
|
||||
$(e).remove();
|
||||
});
|
||||
|
||||
// Remove unwanted DOMs
|
||||
const unwanted_element_selectors = ['amp-ad', '.wsj-ad'];
|
||||
unwanted_element_selectors.forEach((selector) => {
|
||||
content.find(selector).each((i, e) => {
|
||||
$(e).remove();
|
||||
});
|
||||
});
|
||||
|
||||
// Paywall
|
||||
content.find('.paywall').each((i, e) => {
|
||||
// Caption follows, no need to handle caption
|
||||
$(e.childNodes).insertBefore(e);
|
||||
$(e).remove();
|
||||
});
|
||||
|
||||
return {
|
||||
title: item.title,
|
||||
id: item.guid,
|
||||
pubDate: new Date(publishedAt).toUTCString(),
|
||||
updated: new Date(updatedAt).toUTCString(),
|
||||
author: item.creator,
|
||||
link: item.link,
|
||||
summary: summary,
|
||||
description: content.html(),
|
||||
category: categories,
|
||||
icon: 'https://s.wsj.net/media/wsj_launcher-icon-4x.png',
|
||||
logo: 'https://vir.wsj.net/fp/assets/webpack4/img/wsj-logo-big-black.165e51cc.svg',
|
||||
};
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
ctx.state.data = {
|
||||
title: categoryToName[category],
|
||||
link: feed.link,
|
||||
description: feed.description,
|
||||
item: items,
|
||||
language: feed.language,
|
||||
icon: 'https://s.wsj.net/media/wsj_launcher-icon-4x.png',
|
||||
logo: 'https://vir.wsj.net/fp/assets/webpack4/img/wsj-logo-big-black.165e51cc.svg',
|
||||
};
|
||||
};
|
||||
Loading…
Reference in New Issue