diff --git a/docs/en/finance.md b/docs/en/finance.md index 8ec85407c..6bbfffacc 100644 --- a/docs/en/finance.md +++ b/docs/en/finance.md @@ -4,6 +4,29 @@ pageClass: routes # Finance +## Bloomberg + +### News + + + +| Site | Name | +| ---- | ---- | +| / | News | +| bpol | Politics | +| bbiz | Business | +| markets | Markets | +| technology | Technology | +| green | Green | +| wealth | Wealth | +| pursuits | Pursuits | +| bview | Opinion | +| equality | Equality | +| businessweek | Businessweek | +| citylab | CityLab | + + + ## CFD ### Indices Dividend Adjustment (GBP) diff --git a/lib/v2/bloomberg/index.js b/lib/v2/bloomberg/index.js new file mode 100644 index 000000000..4ac3c0889 --- /dev/null +++ b/lib/v2/bloomberg/index.js @@ -0,0 +1,37 @@ +const { rootUrl, parseNewsList, parseArticle } = require('./utils'); +const asyncPool = require('tiny-async-pool'); +const site_title_mapping = { + '/': 'News', + bpol: 'Politics', + bbiz: 'Business', + markets: 'Markets', + technology: 'Technology', + green: 'Green', + wealth: 'Wealth', + pursuits: 'Pursuits', + bview: 'Opinion', + equality: 'Equality', + businessweek: 'Businessweek', + citylab: 'CityLab', +}; + +const asyncPoolAll = async (...args) => { + const results = []; + for await (const result of asyncPool(...args)) { + results.push(result); + } + return results; +}; + +module.exports = async (ctx) => { + const { site } = ctx.params; + const currentUrl = site ? `${rootUrl}/${site}/sitemap_news.xml` : `${rootUrl}/sitemap_news.xml`; + + const list = await parseNewsList(currentUrl, ctx); + const items = await asyncPoolAll(1, list, (item) => parseArticle(item, ctx)); + ctx.state.data = { + title: `Bloomberg - ${site_title_mapping[site ?? '/']}`, + link: currentUrl, + item: items, + }; +}; diff --git a/lib/v2/bloomberg/maintainer.js b/lib/v2/bloomberg/maintainer.js new file mode 100644 index 000000000..b0f1e5315 --- /dev/null +++ b/lib/v2/bloomberg/maintainer.js @@ -0,0 +1,4 @@ +module.exports = { + '/:site?': ['bigfei'], + '/': ['bigfei'], +}; diff --git a/lib/v2/bloomberg/radar.js b/lib/v2/bloomberg/radar.js new file mode 100644 index 000000000..f91cca2a1 --- /dev/null +++ b/lib/v2/bloomberg/radar.js @@ -0,0 +1,13 @@ +module.exports = { + 'bloomberg.com': { + _name: 'bloomberg', + www: [ + { + title: 'Bloomberg', + docs: 'https://docs.rsshub.app/finance.html##bloomberg-news', + source: ['/:site', '/'], + target: '/bloomberg/:site?', + }, + ], + }, +}; diff --git a/lib/v2/bloomberg/router.js b/lib/v2/bloomberg/router.js new file mode 100644 index 000000000..12005eff6 --- /dev/null +++ b/lib/v2/bloomberg/router.js @@ -0,0 +1,4 @@ +module.exports = function (router) { + router.get('/:site', require('./index')); + router.get('/', require('./index')); +}; diff --git a/lib/v2/bloomberg/templates/audio_media.art b/lib/v2/bloomberg/templates/audio_media.art new file mode 100644 index 000000000..8aee10132 --- /dev/null +++ b/lib/v2/bloomberg/templates/audio_media.art @@ -0,0 +1,13 @@ +
+
+ + +
+
+
{{@ caption }}
+
{{@ credit }}
+
+
diff --git a/lib/v2/bloomberg/templates/image_figure.art b/lib/v2/bloomberg/templates/image_figure.art new file mode 100644 index 000000000..365bf7352 --- /dev/null +++ b/lib/v2/bloomberg/templates/image_figure.art @@ -0,0 +1,9 @@ +
+ {{ alt }} + {{if caption}} +
+
{{@ caption }}
+
{{@ credit }}
+
+ {{/if}} +
diff --git a/lib/v2/bloomberg/templates/lede_media.art b/lib/v2/bloomberg/templates/lede_media.art new file mode 100644 index 000000000..4acd0cd7a --- /dev/null +++ b/lib/v2/bloomberg/templates/lede_media.art @@ -0,0 +1,14 @@ +{{if (media.kind =='image') }} +
+ {{ media.description }} + {{if media.caption}} +
+
{{@ media.caption }}
+
{{@ media.credit }}
+
+ {{/if}} +
+{{/if}} +{{if (media.kind =='video') }} +{{include './video_media.art' media.video}} +{{/if}} diff --git a/lib/v2/bloomberg/templates/video_media.art b/lib/v2/bloomberg/templates/video_media.art new file mode 100644 index 000000000..355048bb8 --- /dev/null +++ b/lib/v2/bloomberg/templates/video_media.art @@ -0,0 +1,25 @@ +
+ + {{if caption}} +
+
{{@ caption }}
+
+ {{/if}} +
diff --git a/lib/v2/bloomberg/utils.js b/lib/v2/bloomberg/utils.js new file mode 100644 index 000000000..e8ce1f4f3 --- /dev/null +++ b/lib/v2/bloomberg/utils.js @@ -0,0 +1,395 @@ +const cheerio = require('cheerio'); +const path = require('path'); + +const { parseDate } = require('@/utils/parse-date'); +const got = require('@/utils/got'); +const { art } = require('@/utils/render'); + +const rootUrl = 'https://www.bloomberg.com/feeds'; +const sel = 'script[data-component-props="ArticleBody"], script[data-component-props="FeatureBody"]'; +const apiEndpoints = { + articles: { + url: 'https://www.bloomberg.com/javelin/api/foundation_transporter/', + sel, + }, + features: { + url: 'https://www.bloomberg.com/javelin/api/foundation_feature_transporter/', + sel, + }, + audio: { + url: 'https://www.bloomberg.com/news/audio/', + sel: 'script#__NEXT_DATA__', + }, + videos: { + url: 'https://www.bloomberg.com/news/videos/', + sel: '.footer-container + script', + }, + newsletters: { + url: 'https://www.bloomberg.com/news/newsletters/', + sel, + }, + 'photo-essays': { + url: 'https://www.bloomberg.com/javelin/api/photo-essay_transporter/', + sel: 'script[type = "application/json"][data-component-props]', + }, + 'features/': { + url: 'https://www.bloomberg.com/features/', + sel: 'script#__SSR_DATA__', + }, +}; +const headers = { + accept: 'application/json', + 'cache-control': 'no-cache', + referer: 'https://www.bloomberg.com', +}; + +const pageTypeRegex1 = /\/(?[\w-]*?)\/(?\d{4}-\d{2}-\d{2}\/.*)/; +const pageTypeRegex2 = /(?features\/|graphics\/)(?.*)/; +const regex = [pageTypeRegex1, pageTypeRegex2]; + +const capRegex = /

|<\/p>/g; +const emptyRegex = /]*>( |\s)<\/p>/g; + +const parseNewsList = async (url, ctx) => { + const resp = await got(url); + const $ = cheerio.load(resp.data, { + xml: { + xmlMode: true, + }, + }); + const urls = $('urlset url'); + return urls + .toArray() + .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 50) + .map((u) => { + u = $(u); + const item = { + title: u.find('news\\:title').text(), + link: u.find('loc').text(), + pubDate: parseDate(u.find('news\\:publication_date').text()), + }; + return item; + }); +}; + +const parseArticle = async (item, ctx) => + await ctx.cache.tryGet(item.link, async () => { + const group = regex + .map((r) => r.exec(item.link)) + .filter((e) => e && e.groups) + .map((a) => a && a.groups)[0]; + if (group) { + const { page, link } = group; + + if (apiEndpoints[page]) { + const api = apiEndpoints[page]; + const apiUrl = `${api.url}${link}`; + let res; + try { + res = await got(apiUrl, { headers }); + } catch (err) { + // fallback + if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) { + try { + res = await got(item.link, { headers }); + } catch (err) { + // return the default one + return { + title: item.title, + link: item.link, + pubDate: item.pubDate, + }; + } + } + } + + // Blocked by PX3, return the default + const redirectUrls = res.redirectUrls.map(String); + if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html')) { + return { + title: item.title, + link: item.link, + pubDate: item.pubDate, + }; + } + + switch (page) { + case 'audio': + return await parseAudioPage(res, api, item); + case 'videos': + return await parseVideoPage(res, api, item); + case 'photo-essays': + return await parsePhotoEssaysPage(res, api, item); + case 'features/': // single features page + return await parseFeaturePage(res, api, item); + default: + return await parseOtherPage(res, api, item); + } + } + } + return item; + }); + +const parseAudioPage = async (res, api, item) => { + const audio_json = JSON.parse(cheerio.load(res.data)(api.sel).html()).props.pageProps; + const episode = audio_json.episode; + const rss_item = { + title: episode.title || item.title, + link: audio_json.pageInfo.canonicalUrl || item.link, + guid: `bloomberg:${episode.id}`, + description: (await processBody(episode.articleBody, audio_json)).replaceAll(emptyRegex, ''), + pubDate: parseDate(episode.publishedAt) || item.pubDate, + author: audio_json.hero.showTitle, + media: { + content: { url: episode.image }, + thumbnails: { url: episode.image }, + }, + enclosure_type: 'audio/mpeg', + enclosure_url: episode.url, + itunes_item_image: episode.image || audio_json.pageInfo.image.url, + }; + return rss_item; +}; + +const parseVideoPage = async (res, api, item) => { + const json = cheerio.load(res.data)(api.sel).text().trim().replace('window.__PRELOADED_STATE__ = ', '').slice(0, -1); + const article_json = JSON.parse(json); + const video_story = article_json.video?.videoStory ?? article_json.quicktakeVideo?.videoStory; + if (video_story) { + const desc = await processVideo(video_story.video.bmmrId, video_story.summary.html.replaceAll(emptyRegex, '')); + const rss_item = { + title: video_story.headline.text || item.title, + link: video_story.url || item.link, + guid: `bloomberg:${video_story.id}`, + description: art(path.join(__dirname, 'templates/video_media.art'), desc), + pubDate: parseDate(video_story.publishedAt) || item.pubDate, + media: { + content: { url: video_story.video?.thumbnail.url || '' }, + thumbnails: { url: video_story.video?.thumbnail.url || '' }, + }, + category: desc.keywords ?? [], + }; + return rss_item; + } + return item; +}; + +const parsePhotoEssaysPage = async (res, api, item) => { + const $ = cheerio.load(res.data.html); + const article_json = $(api.sel) + .toArray() + .map((e) => JSON.parse($(e).html())) + .reduce((pv, cv) => ({ ...pv, ...cv }), {}); + const rss_item = { + title: article_json.headline || item.title, + link: article_json.canonical || item.link, + guid: `bloomberg:${article_json.id}`, + description: (await processBody(article_json.body, article_json)).replaceAll(emptyRegex, ''), + pubDate: item.pubDate, + author: article_json.authors?.map((a) => a.name).join(', ') ?? [], + }; + return rss_item; +}; + +const parseFeaturePage = async (res, api, item) => { + const json = cheerio.load(res.data)(api.sel).text().trim().split('\n')[0].replace('var ds__data = ', '').slice(0, -1); + const article_json = JSON.parse(JSON.parse(json)); + const meta = article_json.meta; + + const desc = article_json.blocks.map(async (b) => { + if (b.type === 'Paragraph') { + return b.props.text.html; + } + if (b.type === 'ImageBlock') { + const image = { src: b.props?.src || '', alt: b.props?.alt || '', caption: b.props?.caption || '', credit: b.props?.credit || '' }; + return art(path.join(__dirname, 'templates/image_figure.art'), image); + } + if (b.type === 'Lede') { + return await processLedeMedia(b); + } + return ''; + }); + const rss_item = { + title: meta.title || item.title, + link: item.link, + description: (await Promise.all(desc)).join(''), + pubDate: parseDate(meta.pubDateTime) || item.pubDate, + author: meta.authors?.map((a) => a.name).join(', ') ?? [], + media: { + content: { url: meta.socialImage?.url ?? '' }, + thumbnails: { url: meta.socialImage?.url ?? '' }, + }, + category: + meta.keywords + .split(',') + .map((e) => e.trim()) + .filter((e) => !!e) ?? [], + }; + return rss_item; +}; + +const parseOtherPage = async function (res, api, item) { + const article_json = JSON.parse( + cheerio + .load(res.data.html ?? res.data)(api.sel) + .html() + ); + const story_json = article_json.story; + const body_html = story_json.body; + const media_img = story_json.ledeImageUrl || Object.values(story_json.imageAttachments ?? {})[0]?.baseUrl; + + const rss_item = { + title: story_json.textHeadline || item.title, + link: story_json.canonical || item.link, + guid: `bloomberg:${story_json.id}`, + description: (processHeadline(story_json) + (await processLedeMedia(story_json)) + (await processBody(body_html, story_json))).replaceAll(emptyRegex, ''), + pubDate: parseDate(story_json.publishedAt) || item.pubDate, + author: story_json.authors?.map((a) => a.name).join(', ') ?? [], + category: story_json.mostRelevantTags ?? [], + media: { + content: { url: media_img }, + thumbnails: { url: media_img }, + }, + }; + return rss_item; +}; + +const processHeadline = (story_json) => { + const dek = story_json.dek || ''; + const abs = story_json.abstract?.map((a) => `

  • ${a}
    • `).join(''); + return abs ? dek + `` : dek; +}; + +const processLedeMedia = async (story_json) => { + if (story_json.ledeMediaKind) { + const kind = story_json.ledeMediaKind; + + const media = { + kind: story_json.ledeMediaKind, + caption: story_json.ledeCaption?.replaceAll(capRegex, '') ?? '', + description: story_json.ledeDescription?.replaceAll(capRegex, '') ?? '', + credit: story_json.ledeCredit?.replaceAll(capRegex, '') ?? '', + src: story_json.ledeImageUrl, + video: kind === 'video' && (await processVideo(story_json.ledeAttachment.bmmrId)), + }; + return art(path.join(__dirname, 'templates/lede_media.art'), { media }); + } else if (story_json.imageAttachments) { + const attachment = Object.values(story_json.imageAttachments)[0]; + if (attachment) { + const image = { + src: attachment.baseUrl, + alt: attachment.alt, + }; + return art(path.join(__dirname, 'templates/image_figure.art'), image); + } + return ''; + } else if (story_json.type === 'Lede') { + const props = story_json.props; + + const media = { + kind: props.media, + caption: props.caption?.replaceAll(capRegex, '') ?? '', + description: props.dek?.replaceAll(capRegex, '') ?? '', + credit: props.credit?.replaceAll(capRegex, '') ?? '', + src: props.url, + }; + return art(path.join(__dirname, 'templates/lede_media.art'), { media }); + } +}; + +const processVideo = async (bmmrId, summary) => { + const api = `https://www.bloomberg.com/multimedia/api/embed?id=${bmmrId}`; + const res = await got(api, { headers }); + + // Blocked by PX3, return the default + const redirectUrls = res.redirectUrls.map(String); + if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html')) { + return { + stream: '', + mp4: '', + coverUrl: '', + caption: summary, + }; + } + + if (res.data) { + const video_json = res.data; + return { + stream: video_json.streams ? video_json.streams[0]?.url : '', + mp4: video_json.downloadURLs ? video_json.downloadURLs['600'] : '', + coverUrl: video_json.thumbnail?.baseUrl ?? '', + caption: video_json.description || video_json.title || summary, + }; + } + return {}; +}; + +const processBody = async (body_html, story_json) => { + const removeSel = ['meta', 'script', '*[class$="-footnotes"]', '*[class$="for-you"]', '*[class$="-newsletter"]', '*[class$="page-ad"]', '*[class$="-recirc"]', '*[data-ad-placeholder="Advertisement"]']; + + const $ = cheerio.load(body_html); + removeSel.forEach((sel) => $(sel).remove()); + $('.paywall').removeAttr('class'); + + // Asynchronous iteration intentionally + // https://github.com/eslint/eslint/blob/8a159686f9d497262d573dd601855ce28362199b/tests/lib/rules/no-await-in-loop.js#L50 + for await (const e of $('figure')) { + const imageType = $(e).data('image-type'); + const type = $(e).data('type'); + + let new_figure = ''; + if (imageType === 'audio') { + let audio = {}; + if (story_json.audios) { + const attachment = story_json.audios.find((a) => a.id.toString() === $(e).data('id').toString()); + audio = { + img: attachment.image?.url || $(e).find('img').attr('src'), + src: attachment.url || $(e).find('audio source').attr('src'), + caption: $(e).find('[class$="text"]').html()?.trim() ?? '', + credit: $(e).find('[class$="credit"]').html()?.trim() ?? '', + }; + } + if (story_json.episode) { + const episode = story_json.episode; + audio = { + src: episode.url, + img: episode.image || story_json.pageInfo.image.url, + + caption: episode.description || ($(e).find('[class$="text"]').html()?.trim() ?? ''), + credit: (episode.credits.map((c) => c.name).join(', ') ?? []) || ($(e).find('[class$="credit"]').html()?.trim() ?? ''), + }; + } + new_figure = art(path.join(__dirname, 'templates/audio_media.art'), audio); + } else if (imageType === 'video') { + if (story_json.videoAttachments) { + const attachment = story_json.videoAttachments[$(e).data('id')]; + const video = await processVideo(attachment.bmmrId); + new_figure = art(path.join(__dirname, 'templates/video_media.art'), video); + } + } else if (imageType === 'photo' || imageType === 'image' || type === 'image') { + let src, alt; + if (story_json.imageAttachments) { + const attachment = story_json.imageAttachments[$(e).data('id')]; + alt = attachment?.alt || $(e).find('img').attr('alt')?.trim(); + src = attachment?.baseUrl; + } else { + alt = $(e).find('img').attr('alt').trim(); + src = $(e).find('img').data('native-src'); + } + const caption = $(e).find('[class$="text"], .caption, .photo-essay__text').html()?.trim() ?? ''; + const credit = $(e).find('[class$="credit"], .credit, .photo-essay__source').html()?.trim() ?? ''; + const image = { src, alt, caption, credit }; + new_figure = art(path.join(__dirname, 'templates/image_figure.art'), image); + } + $(new_figure).insertAfter(e); + $(e).remove(); + } + + return $.html(); +}; + +module.exports = { + rootUrl, + parseNewsList, + parseArticle, +};