diff --git a/lib/v2/bloomberg/react-renderer.js b/lib/v2/bloomberg/react-renderer.js
deleted file mode 100644
index 2298f1bc6..000000000
--- a/lib/v2/bloomberg/react-renderer.js
+++ /dev/null
@@ -1,225 +0,0 @@
-const art = require('art-template');
-const path = require('path');
-const got = require('@/utils/got');
-
-const headers = {
- accept: 'application/json',
- 'cache-control': 'no-cache',
- referer: 'https://www.bloomberg.com',
-};
-
-const processVideo = async (bmmrId, summary) => {
- const api = `https://www.bloomberg.com/multimedia/api/embed?id=${bmmrId}`;
- const res = await got(api, { headers });
-
- // Blocked by PX3, return the default
- const redirectUrls = res.redirectUrls.map(String);
- if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html')) {
- return {
- stream: '',
- mp4: '',
- coverUrl: '',
- caption: summary,
- };
- }
-
- if (res.data) {
- const video_json = res.data;
- return {
- stream: video_json.streams ? video_json.streams[0]?.url : '',
- mp4: video_json.downloadURLs ? video_json.downloadURLs['600'] : '',
- coverUrl: video_json.thumbnail?.baseUrl ?? '',
- caption: video_json.description || video_json.title || summary,
- };
- }
- return {};
-};
-
-const nodeRenderers = {
- paragraph: async (node, nextNode) => `${await nextNode(node.content)}
`,
- text: (node) => {
- const { attributes: attr, value: val } = node;
- if (attr?.emphasis && attr?.strong) {
- return `${val}`;
- } else if (attr?.emphasis) {
- return `${val}`;
- } else if (attr?.strong) {
- return `${val}`;
- } else {
- return val;
- }
- },
- 'inline-newsletter': async (node, nextNode) => `${await nextNode(node.content)}
`,
- heading: async (node, nextNode) => {
- const nodeData = node.data;
- if (nodeData.level === 2 || nodeData.level === 3) {
- return `${await nextNode(node.content)}
`;
- }
- },
- link: async (node, nextNode) => {
- const dest = node.data.destination;
- const web = dest.web;
- const bbg = dest.bbg;
- const title = node.data.title;
- if (web) {
- return `${await nextNode(node.content)}`;
- }
-
- if (bbg && bbg.startsWith('bbg://news/stories')) {
- const o = bbg.split('bbg://news/stories/').pop();
- const s = 'https://www.bloomberg.com/news/terminal/'.concat(o);
- return `${await nextNode(node.content)}`;
- }
- return String(await nextNode(node.content));
- },
- entity: async (node, nextNode) => {
- const t = node.subType;
- const linkDest = node.data.link.destination;
- const web = linkDest.web;
- if (t === 'person') {
- return nextNode(node.content);
- }
- if (t === 'story') {
- if (web) {
- return `${await nextNode(node.content)}`;
- }
- const a = node.data.story.identifiers.suid;
- const o = 'https://www.bloomberg.com/news/terminal/'.concat(a);
- return `${await nextNode(node.content)}`;
- }
- if (t === 'security') {
- const s = node.data.security.identifiers.parsekey;
- if (s) {
- const c = s.split(' ');
- const href = 'https://www.bloomberg.com/quote/'.concat(c[0], ':').concat(c[1]);
- return `${await nextNode(node.content)}`;
- }
- }
- return nextNode(node.content);
- },
- br: () => `
`,
- hr: () => `
`,
- ad: () => {},
- blockquote: async (node, nextNode) => `${await nextNode(node.content)}
`,
- quote: async (node, nextNode) => `${await nextNode(node.content)}
`,
- aside: async (node, nextNode) => ``,
- list: async (node, nextNode) => {
- const t = node.subType;
- if (t === 'unordered') {
- return `${await nextNode(node.content)}
`;
- }
- if (t === 'ordered') {
- return `${await nextNode(node.content)}
`;
- }
- },
- listItem: async (node, nextNode) => `${await nextNode(node.content)}`,
- media: async (node) => {
- const t = node.subType;
- if (t === 'chart' && node.data.attachment) {
- if (node.data.attachment.creator === 'TOASTER') {
- const c = node.data.chart;
- const e = {
- src: (c && c.fallback) || '',
- chart: node.data.attachment,
- id: (c && c.id) || '',
- alt: (c && c.alt) || '',
- };
- const w = e.chart;
-
- const chart = {
- source: w.source,
- footnote: w.footnote,
- url: w.url,
- title: w.title,
- subtitle: w.subtitle,
- chartId: 'toaster-chart-'.concat(e.id),
- chartAlt: e.alt,
- fallback: e.src,
- };
- return art(path.join(__dirname, 'templates/chart_media.art'), { chart });
- }
- const image = {
- alt: node.data.attachment?.footnote || '',
- caption: node.data.attachment?.title + node.data.attachment.subtitle || '',
- credit: node.data.attachment?.source || '',
- src: node.data.chart?.fallback || '',
- };
- return art(path.join(__dirname, 'templates/image_figure.art'), image);
- }
- if (t === 'photo') {
- const h = node.data;
- let img = '';
- if (h.attachment) {
- const image = { src: h.photo?.src, alt: h.photo?.alt, caption: h.photo?.caption, credit: h.photo?.credit };
- img = art(path.join(__dirname, 'templates/image_figure.art'), image);
- }
- if (h.link && h.link.destination && h.link.destination.web) {
- const href = h.link.destination.web;
- return `${img}`;
- }
- return img;
- }
- if (t === 'video') {
- const h = node.data;
- const id = h.attachment?.id;
- const desc = await processVideo(id, h.attachment?.title);
- return art(path.join(__dirname, 'templates/video_media.art'), desc);
- }
- if (t === 'audio' && node.data.attachment) {
- const B = node.data.attachment;
- const P = B.title;
- const D = B.url;
- const M = B.image;
- if (P && D) {
- const audio = {
- src: D,
- img: M,
- caption: P,
- credit: '',
- };
- return art(path.join(__dirname, 'templates/audio_media.art'), audio);
- }
- }
- return '';
- },
-};
-
-const nodeToHtmlString = async (node, obj) => {
- const nextNode = async (nodes) => {
- const nodeStr = await nodeListToHtmlString(nodes);
- return nodeStr;
- };
- if (!node.type || !nodeRenderers[node.type]) {
- return `${node.type}`;
- }
- const str = await nodeRenderers[node.type](node, nextNode, obj);
- return str;
-};
-
-const nodeListToHtmlString = async (nodes) => {
- const res = await Promise.all(
- nodes.map(async (node, index) => {
- const str = await nodeToHtmlString(node, {
- index,
- prev: nodes[index - 1]?.type,
- next: nodes[index + 1]?.type,
- });
- return str;
- })
- );
- return res.join('');
-};
-
-const documentToHtmlString = async (document) => {
- if (!document || !document.content) {
- return '';
- }
- const str = await nodeListToHtmlString(document.content);
- return str;
-};
-
-module.exports = {
- documentToHtmlString,
- processVideo,
- headers,
-};
diff --git a/lib/v2/bloomberg/utils.js b/lib/v2/bloomberg/utils.js
index 419194459..44f193c34 100644
--- a/lib/v2/bloomberg/utils.js
+++ b/lib/v2/bloomberg/utils.js
@@ -6,20 +6,26 @@ const { parseDate } = require('@/utils/parse-date');
const got = require('@/utils/got');
const { art } = require('@/utils/render');
-const { documentToHtmlString, processVideo, headers } = require('./react-renderer');
-
const rootUrl = 'https://www.bloomberg.com/feeds';
-const sel = 'script[data-component-props="ArticleBody"], script[data-component-props="FeatureBody"]';
+const idSel = 'script[id^="article-info"][type="application/json"], script[class^="article-info"][type="application/json"]';
+const idUrl = 'https://www.bloomberg.com/article/api/story/id/';
+
+const headers = {
+ accept: 'application/json',
+ 'cache-control': 'no-cache',
+ referer: 'https://www.bloomberg.com',
+};
+
const apiEndpoints = {
articles: {
- url: 'https://www.bloomberg.com/javelin/api/foundation_transporter/',
- sel,
+ url: 'https://www.bloomberg.com/article/api/story/slug/',
},
features: {
- url: 'https://www.bloomberg.com/javelin/api/foundation_feature_transporter/',
- sel,
+ // https://www.bloomberg.com/news/features/2023-08-12/boston-university-data-science-hub-is-a-textbook-example-of-jenga-architecture
+ url: 'https://www.bloomberg.com/article/api/story/slug/',
},
audio: {
+ // https://www.bloomberg.com/news/audio/2023-07-26/daybreak-deutsche-traders-outperform-as-costs-rise-podcast
url: 'https://www.bloomberg.com/news/audio/',
sel: 'script#__NEXT_DATA__',
},
@@ -28,16 +34,18 @@ const apiEndpoints = {
sel: 'script',
},
newsletters: {
- url: 'https://www.bloomberg.com/news/newsletters/',
- sel: 'script#__NEXT_DATA__',
+ // https://www.bloomberg.com/news/newsletters/2023-07-20/key-votes-the-bloomberg-open-europe-edition
+ url: 'https://www.bloomberg.com/article/api/story/slug/',
},
'photo-essays': {
url: 'https://www.bloomberg.com/javelin/api/photo-essay_transporter/',
sel: 'script[type = "application/json"][data-component-props]',
},
'features/': {
+ // https://www.bloomberg.com/features/2023-stradivarius-murders/
url: 'https://www.bloomberg.com/features/',
- sel: 'script#__SSR_DATA__',
+ sel: idSel,
+ props: 'id',
},
};
@@ -90,7 +98,6 @@ const parseArticle = (item, ctx) =>
if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) {
try {
res = await got(item.link, { headers });
- api.useOrigLink = true;
} catch (err) {
// return the default one
return {
@@ -120,38 +127,15 @@ const parseArticle = (item, ctx) =>
case 'photo-essays':
return parsePhotoEssaysPage(res, api, item);
case 'features/': // single features page
- return parseFeaturePage(res, api, item);
- case 'newsletters':
- return parseNewsletterPage(res, api, item);
- default:
- return parseOtherPage(res, api, item);
+ return parseReactRendererPage(res, api, item);
+ default: // use story api to get json
+ return parseStoryJson(res.data, item);
}
}
}
return item;
});
-const parseNewsletterPage = async (res, api, item) => {
- const newsletter_json = JSON.parse(cheerio.load(res.data)(api.sel).html()).props.pageProps;
- const story_json = newsletter_json.story;
- const media_img = story_json.ledeImageUrl || Object.values(story_json.imageAttachments ?? {})[0]?.baseUrl;
-
- const rss_item = {
- title: story_json.headline || item.title,
- link: story_json.url || item.link,
- guid: `bloomberg:${story_json.id}`,
- description: processHeadline(story_json) + (await processLedeMedia(story_json)) + (await documentToHtmlString(story_json.body || '')),
- pubDate: parseDate(story_json.publishedAt) || item.pubDate,
- author: story_json.authors?.map((a) => a.name).join(', ') ?? [],
- category: story_json.mostRelevantTags ?? [],
- media: {
- content: { url: media_img },
- thumbnails: { url: media_img },
- },
- };
- return rss_item;
-};
-
const parseAudioPage = async (res, api, item) => {
const audio_json = JSON.parse(cheerio.load(res.data)(api.sel).html()).props.pageProps;
const episode = audio_json.episode;
@@ -215,62 +199,31 @@ const parsePhotoEssaysPage = async (res, api, item) => {
return rss_item;
};
-const parseFeaturePage = async (res, api, item) => {
- const json = cheerio.load(res.data)(api.sel).text().trim().split('\n')[0].replace('var ds__data = ', '').slice(0, -1);
- const article_json = JSON.parse(JSON.parse(json));
- const meta = article_json.meta;
-
- const desc = article_json.blocks.map((b) => {
- if (b.type === 'Paragraph') {
- return b.props.text.html;
+const parseReactRendererPage = async (res, api, item) => {
+ const json = cheerio.load(res.data)(api.sel).text().trim();
+ const story_id = JSON.parse(json)[api.prop];
+ try {
+ const res = await got(`${idUrl}${story_id}`, { headers });
+ return await parseStoryJson(res.data, item);
+ } catch (err) {
+ // fallback
+ if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) {
+ return {
+ title: item.title,
+ link: item.link,
+ pubDate: item.pubDate,
+ };
}
- if (b.type === 'ImageBlock') {
- const image = { src: b.props?.src || '', alt: b.props?.alt || '', caption: b.props?.caption || '', credit: b.props?.credit || '' };
- return art(path.join(__dirname, 'templates/image_figure.art'), image);
- }
- if (b.type === 'Lede') {
- return processLedeMedia(b);
- }
- return '';
- });
- const rss_item = {
- title: meta.title || item.title,
- link: item.link,
- description: (await Promise.all(desc)).join(''),
- pubDate: parseDate(meta.pubDateTime) || item.pubDate,
- author: meta.authors?.map((a) => a.name).join(', ') ?? [],
- media: {
- content: { url: meta.socialImage?.url ?? '' },
- thumbnails: { url: meta.socialImage?.url ?? '' },
- },
- category:
- meta.keywords
- .split(',')
- .map((e) => e.trim())
- .filter((e) => !!e) ?? [],
- };
- return rss_item;
+ }
};
-const parseOtherPage = async function (res, api, item) {
- if (api.useOrigLink) {
- return parseNewsletterPage(res, apiEndpoints.newsletters, item);
- }
- const article_json = JSON.parse(
- cheerio
- .load(res.data.html ?? res.data)(api.sel)
- .html()
- );
-
- const story_json = article_json.story;
- const body_html = story_json.body;
- const media_img = story_json.ledeImageUrl || Object.values(story_json.imageAttachments ?? {})[0]?.baseUrl;
-
+const parseStoryJson = async (story_json, item) => {
+ const media_img = story_json.ledeImageUrl || Object.values(story_json.imageAttachments ?? {})[0]?.url;
const rss_item = {
- title: story_json.textHeadline || item.title,
- link: story_json.canonical || item.link,
+ title: story_json.headline || item.title,
+ link: story_json.url || item.link,
guid: `bloomberg:${story_json.id}`,
- description: (processHeadline(story_json) + (await processLedeMedia(story_json)) + (await processBody(body_html, story_json))).replaceAll(emptyRegex, ''),
+ description: processHeadline(story_json) + (await processLedeMedia(story_json)) + (await documentToHtmlString(story_json.body || '')),
pubDate: parseDate(story_json.publishedAt) || item.pubDate,
author: story_json.authors?.map((a) => a.name).join(', ') ?? [],
category: story_json.mostRelevantTags ?? [],
@@ -306,6 +259,8 @@ const processLedeMedia = async (story_json) => {
const image = {
src: lede.url,
alt: lede.alt || lede.title,
+ caption: lede.caption?.replaceAll(capRegex, '') ?? '',
+ credit: lede.credit?.replaceAll(capRegex, '') ?? '',
};
return art(path.join(__dirname, 'templates/image_figure.art'), image);
} else if (story_json.imageAttachments) {
@@ -314,6 +269,8 @@ const processLedeMedia = async (story_json) => {
const image = {
src: attachment.baseUrl || attachment.url,
alt: attachment.alt || attachment.title,
+ caption: attachment.caption?.replaceAll(capRegex, '') ?? '',
+ credit: attachment.credit?.replaceAll(capRegex, '') ?? '',
};
return art(path.join(__dirname, 'templates/image_figure.art'), image);
}
@@ -396,6 +353,234 @@ const processBody = async (body_html, story_json) => {
return $.html();
};
+const processVideo = async (bmmrId, summary) => {
+ const api = `https://www.bloomberg.com/multimedia/api/embed?id=${bmmrId}`;
+ const res = await got(api, { headers });
+
+ // Blocked by PX3, return the default
+ const redirectUrls = res.redirectUrls.map(String);
+ if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html')) {
+ return {
+ stream: '',
+ mp4: '',
+ coverUrl: '',
+ caption: summary,
+ };
+ }
+
+ if (res.data) {
+ const video_json = res.data;
+ return {
+ stream: video_json.streams ? video_json.streams[0]?.url : '',
+ mp4: video_json.downloadURLs ? video_json.downloadURLs['600'] : '',
+ coverUrl: video_json.thumbnail?.baseUrl ?? '',
+ caption: video_json.description || video_json.title || summary,
+ };
+ }
+ return {};
+};
+
+const nodeRenderers = {
+ paragraph: async (node, nextNode) => `${await nextNode(node.content)}
`,
+ text: (node) => {
+ const { attributes: attr, value: val } = node;
+ if (attr?.emphasis && attr?.strong) {
+ return `${val}`;
+ } else if (attr?.emphasis) {
+ return `${val}`;
+ } else if (attr?.strong) {
+ return `${val}`;
+ } else {
+ return val;
+ }
+ },
+ 'inline-newsletter': async (node, nextNode) => `${await nextNode(node.content)}
`,
+ 'inline-recirc': async (node, nextNode) => `${await nextNode(node.content)}
`,
+ heading: async (node, nextNode) => {
+ const nodeData = node.data;
+ if (nodeData.level === 2 || nodeData.level === 3) {
+ return `${await nextNode(node.content)}
`;
+ }
+ },
+ link: async (node, nextNode) => {
+ const dest = node.data.destination;
+ const web = dest.web;
+ const bbg = dest.bbg;
+ const title = node.data.title;
+ if (web) {
+ return `${await nextNode(node.content)}`;
+ }
+
+ if (bbg && bbg.startsWith('bbg://news/stories')) {
+ const o = bbg.split('bbg://news/stories/').pop();
+ const s = 'https://www.bloomberg.com/news/terminal/'.concat(o);
+ return `${await nextNode(node.content)}`;
+ }
+ return String(await nextNode(node.content));
+ },
+ entity: async (node, nextNode) => {
+ const t = node.subType;
+ const linkDest = node.data.link.destination;
+ const web = linkDest.web;
+ if (t === 'person') {
+ return nextNode(node.content);
+ }
+ if (t === 'story') {
+ if (web) {
+ return `${await nextNode(node.content)}`;
+ }
+ const a = node.data.story.identifiers.suid;
+ const o = 'https://www.bloomberg.com/news/terminal/'.concat(a);
+ return `${await nextNode(node.content)}`;
+ }
+ if (t === 'security') {
+ const s = node.data.security.identifiers.parsekey;
+ if (s) {
+ const c = s.split(' ');
+ const href = 'https://www.bloomberg.com/quote/'.concat(c[0], ':').concat(c[1]);
+ return `${await nextNode(node.content)}`;
+ }
+ }
+ return nextNode(node.content);
+ },
+ br: () => `
`,
+ hr: () => `
`,
+ ad: () => {},
+ blockquote: async (node, nextNode) => `${await nextNode(node.content)}
`,
+ quote: async (node, nextNode) => `${await nextNode(node.content)}
`,
+ aside: async (node, nextNode) => ``,
+ list: async (node, nextNode) => {
+ const t = node.subType;
+ if (t === 'unordered') {
+ return `${await nextNode(node.content)}
`;
+ }
+ if (t === 'ordered') {
+ return `${await nextNode(node.content)}
`;
+ }
+ },
+ listItem: async (node, nextNode) => `${await nextNode(node.content)}`,
+ media: async (node) => {
+ const t = node.subType;
+ if (t === 'chart' && node.data.attachment) {
+ if (node.data.attachment.creator === 'TOASTER') {
+ const c = node.data.chart;
+ const e = {
+ src: (c && c.fallback) || '',
+ chart: node.data.attachment,
+ id: (c && c.id) || '',
+ alt: (c && c.alt) || '',
+ };
+ const w = e.chart;
+
+ const chart = {
+ source: w.source,
+ footnote: w.footnote,
+ url: w.url,
+ title: w.title,
+ subtitle: w.subtitle,
+ chartId: 'toaster-chart-'.concat(e.id),
+ chartAlt: e.alt,
+ fallback: e.src,
+ };
+ return art(path.join(__dirname, 'templates/chart_media.art'), { chart });
+ }
+ const image = {
+ alt: node.data.attachment?.footnote || '',
+ caption: node.data.attachment?.title + node.data.attachment.subtitle || '',
+ credit: node.data.attachment?.source || '',
+ src: node.data.chart?.fallback || '',
+ };
+ return art(path.join(__dirname, 'templates/image_figure.art'), image);
+ }
+ if (t === 'photo') {
+ const h = node.data;
+ let img = '';
+ if (h.attachment) {
+ const image = { src: h.photo?.src, alt: h.photo?.alt, caption: h.photo?.caption, credit: h.photo?.credit };
+ img = art(path.join(__dirname, 'templates/image_figure.art'), image);
+ }
+ if (h.link && h.link.destination && h.link.destination.web) {
+ const href = h.link.destination.web;
+ return `${img}`;
+ }
+ return img;
+ }
+ if (t === 'video') {
+ const h = node.data;
+ const id = h.attachment?.id;
+ const desc = await processVideo(id, h.attachment?.title);
+ return art(path.join(__dirname, 'templates/video_media.art'), desc);
+ }
+ if (t === 'audio' && node.data.attachment) {
+ const B = node.data.attachment;
+ const P = B.title;
+ const D = B.url;
+ const M = B.image;
+ if (P && D) {
+ const audio = {
+ src: D,
+ img: M.url,
+ caption: P,
+ credit: '',
+ };
+ return art(path.join(__dirname, 'templates/audio_media.art'), audio);
+ }
+ }
+ return '';
+ },
+ tabularData: async (node, nextNode) => `${await nextNode(node.content)}
`,
+ columns: (node) => {
+ const cols = node.data.definitions
+ .map((e) => ({
+ title: e.title,
+ span: e.colSpan || 1,
+ type: e.dataType,
+ }))
+ .map((e) => `${e.title} | `);
+ return `${cols}
`;
+ },
+ row: async (node, nextNode) => `${await nextNode(node.content)}
`,
+ cell: async (node, nextNode) => {
+ const types = { 'news-rsf-table-number': 'number', 'news-rsf-table-string': 'text' };
+ const cellType = types[node.data.class] || 'text';
+ return `${await nextNode(node.content)} | `;
+ },
+};
+
+const nodeToHtmlString = async (node, obj) => {
+ const nextNode = async (nodes) => {
+ const nodeStr = await nodeListToHtmlString(nodes);
+ return nodeStr;
+ };
+ if (!node.type || !nodeRenderers[node.type]) {
+ return `${node.type}`;
+ }
+ const str = await nodeRenderers[node.type](node, nextNode, obj);
+ return str;
+};
+
+const nodeListToHtmlString = async (nodes) => {
+ const res = await Promise.all(
+ nodes.map(async (node, index) => {
+ const str = await nodeToHtmlString(node, {
+ index,
+ prev: nodes[index - 1]?.type,
+ next: nodes[index + 1]?.type,
+ });
+ return str;
+ })
+ );
+ return res.join('');
+};
+
+const documentToHtmlString = async (document) => {
+ if (!document || !document.content) {
+ return '';
+ }
+ const str = await nodeListToHtmlString(document.content);
+ return str;
+};
+
const asyncPoolAll = async (...args) => {
const results = [];
for await (const result of asyncPool(...args)) {