600 lines
23 KiB
JavaScript
600 lines
23 KiB
JavaScript
const cheerio = require('cheerio');
|
|
const path = require('path');
|
|
const asyncPool = require('tiny-async-pool');
|
|
|
|
const { parseDate } = require('@/utils/parse-date');
|
|
const got = require('@/utils/got');
|
|
const { art } = require('@/utils/render');
|
|
|
|
const rootUrl = 'https://www.bloomberg.com/feeds';
|
|
const idSel = 'script[id^="article-info"][type="application/json"], script[class^="article-info"][type="application/json"], script#dvz-config';
|
|
const idUrl = 'https://www.bloomberg.com/article/api/story/id/';
|
|
|
|
const headers = {
|
|
accept: 'application/json',
|
|
'cache-control': 'no-cache',
|
|
referer: 'https://www.bloomberg.com',
|
|
};
|
|
|
|
const apiEndpoints = {
|
|
articles: {
|
|
url: 'https://www.bloomberg.com/article/api/story/slug/',
|
|
},
|
|
features: {
|
|
// https://www.bloomberg.com/news/features/2023-08-12/boston-university-data-science-hub-is-a-textbook-example-of-jenga-architecture
|
|
url: 'https://www.bloomberg.com/article/api/story/slug/',
|
|
},
|
|
audio: {
|
|
// https://www.bloomberg.com/news/audio/2023-07-26/daybreak-deutsche-traders-outperform-as-costs-rise-podcast
|
|
url: 'https://www.bloomberg.com/news/audio/',
|
|
sel: 'script#__NEXT_DATA__',
|
|
},
|
|
videos: {
|
|
url: 'https://www.bloomberg.com/news/videos/',
|
|
sel: 'script',
|
|
},
|
|
newsletters: {
|
|
// https://www.bloomberg.com/news/newsletters/2023-07-20/key-votes-the-bloomberg-open-europe-edition
|
|
url: 'https://www.bloomberg.com/article/api/story/slug/',
|
|
},
|
|
'photo-essays': {
|
|
url: 'https://www.bloomberg.com/javelin/api/photo-essay_transporter/',
|
|
sel: 'script[type = "application/json"][data-component-props]',
|
|
},
|
|
'features/': {
|
|
// https://www.bloomberg.com/features/2023-stradivarius-murders/
|
|
url: 'https://www.bloomberg.com/features/',
|
|
sel: idSel,
|
|
prop: 'id',
|
|
},
|
|
};
|
|
|
|
const pageTypeRegex1 = /\/(?<page>[\w-]*?)\/(?<link>\d{4}-\d{2}-\d{2}\/.*)/;
|
|
const pageTypeRegex2 = /(?<!news|politics)\/(?<page>features\/|graphics\/)(?<link>.*)/;
|
|
const regex = [pageTypeRegex1, pageTypeRegex2];
|
|
|
|
const capRegex = /<p>|<\/p>/g;
|
|
const emptyRegex = /<p\b[^>]*>( |\s)<\/p>/g;
|
|
|
|
const parseNewsList = async (url, ctx) => {
|
|
const resp = await got(url);
|
|
const $ = cheerio.load(resp.data, {
|
|
xml: {
|
|
xmlMode: true,
|
|
},
|
|
});
|
|
const urls = $('urlset url');
|
|
return urls
|
|
.toArray()
|
|
.slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 50)
|
|
.map((u) => {
|
|
u = $(u);
|
|
const item = {
|
|
title: u.find('news\\:title').text(),
|
|
link: u.find('loc').text(),
|
|
pubDate: parseDate(u.find('news\\:publication_date').text()),
|
|
};
|
|
return item;
|
|
});
|
|
};
|
|
|
|
const parseArticle = (item, ctx) =>
|
|
ctx.cache.tryGet(item.link, async () => {
|
|
const group = regex
|
|
.map((r) => r.exec(item.link))
|
|
.filter((e) => e && e.groups)
|
|
.map((a) => a && a.groups)[0];
|
|
if (group) {
|
|
const { page, link } = group;
|
|
if (apiEndpoints[page]) {
|
|
const api = { ...apiEndpoints[page] };
|
|
let res;
|
|
|
|
try {
|
|
const apiUrl = `${api.url}${link}`;
|
|
res = await got(apiUrl, { headers });
|
|
} catch (err) {
|
|
// fallback
|
|
if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) {
|
|
try {
|
|
res = await got(item.link, { headers });
|
|
} catch (err) {
|
|
// return the default one
|
|
return {
|
|
title: item.title,
|
|
link: item.link,
|
|
pubDate: item.pubDate,
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
// Blocked by PX3, or 404 by both api and direct link, return the default
|
|
const redirectUrls = res.redirectUrls.map(String);
|
|
if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html') || res.statusCode === 404) {
|
|
return {
|
|
title: item.title,
|
|
link: item.link,
|
|
pubDate: item.pubDate,
|
|
};
|
|
}
|
|
|
|
switch (page) {
|
|
case 'audio':
|
|
return parseAudioPage(res, api, item);
|
|
case 'videos':
|
|
return parseVideoPage(res, api, item);
|
|
case 'photo-essays':
|
|
return parsePhotoEssaysPage(res, api, item);
|
|
case 'features/': // single features page
|
|
return parseReactRendererPage(res, api, item);
|
|
default: // use story api to get json
|
|
return parseStoryJson(res.data, item);
|
|
}
|
|
}
|
|
}
|
|
return item;
|
|
});
|
|
|
|
const parseAudioPage = async (res, api, item) => {
|
|
const audio_json = JSON.parse(cheerio.load(res.data)(api.sel).html()).props.pageProps;
|
|
const episode = audio_json.episode;
|
|
const rss_item = {
|
|
title: episode.title || item.title,
|
|
link: audio_json.pageInfo.canonicalUrl || item.link,
|
|
guid: `bloomberg:${episode.id}`,
|
|
description: (await processBody(episode.articleBody, audio_json)).replaceAll(emptyRegex, ''),
|
|
pubDate: parseDate(episode.publishedAt) || item.pubDate,
|
|
author: audio_json.hero.showTitle,
|
|
media: {
|
|
content: { url: episode.image },
|
|
thumbnails: { url: episode.image },
|
|
},
|
|
enclosure_type: 'audio/mpeg',
|
|
enclosure_url: episode.url,
|
|
itunes_item_image: episode.image || audio_json.pageInfo.image.url,
|
|
};
|
|
return rss_item;
|
|
};
|
|
|
|
const parseVideoPage = async (res, api, item) => {
|
|
const $ = cheerio.load(res.data);
|
|
const script = $(api.sel).filter((i, el) => $(el).text().includes('__PRELOADED_STATE__'));
|
|
const json = script.text().trim().replace('window.__PRELOADED_STATE__ = ', '').slice(0, -1);
|
|
const article_json = JSON.parse(json);
|
|
const video_story = article_json.video?.videoStory ?? article_json.quicktakeVideo?.videoStory;
|
|
if (video_story) {
|
|
const desc = await processVideo(video_story.video.bmmrId, video_story.summary.html.replaceAll(emptyRegex, ''));
|
|
const rss_item = {
|
|
title: video_story.headline.text || item.title,
|
|
link: video_story.url || item.link,
|
|
guid: `bloomberg:${video_story.id}`,
|
|
description: art(path.join(__dirname, 'templates/video_media.art'), desc),
|
|
pubDate: parseDate(video_story.publishedAt) || item.pubDate,
|
|
media: {
|
|
content: { url: video_story.video?.thumbnail.url || '' },
|
|
thumbnails: { url: video_story.video?.thumbnail.url || '' },
|
|
},
|
|
category: desc.keywords ?? [],
|
|
};
|
|
return rss_item;
|
|
}
|
|
return item;
|
|
};
|
|
|
|
const parsePhotoEssaysPage = async (res, api, item) => {
|
|
const $ = cheerio.load(res.data.html);
|
|
const article_json = $(api.sel)
|
|
.toArray()
|
|
.map((e) => JSON.parse($(e).html()))
|
|
.reduce((pv, cv) => ({ ...pv, ...cv }), {});
|
|
const rss_item = {
|
|
title: article_json.headline || item.title,
|
|
link: article_json.canonical || item.link,
|
|
guid: `bloomberg:${article_json.id}`,
|
|
description: (await processBody(article_json.body, article_json)).replaceAll(emptyRegex, ''),
|
|
pubDate: item.pubDate,
|
|
author: article_json.authors?.map((a) => a.name).join(', ') ?? [],
|
|
};
|
|
return rss_item;
|
|
};
|
|
|
|
const parseReactRendererPage = async (res, api, item) => {
|
|
const json = cheerio.load(res.data)(api.sel).text().trim();
|
|
const story_id = JSON.parse(json)[api.prop];
|
|
try {
|
|
const res = await got(`${idUrl}${story_id}`, { headers });
|
|
return await parseStoryJson(res.data, item);
|
|
} catch (err) {
|
|
// fallback
|
|
if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) {
|
|
return {
|
|
title: item.title,
|
|
link: item.link,
|
|
pubDate: item.pubDate,
|
|
};
|
|
}
|
|
}
|
|
};
|
|
|
|
const parseStoryJson = async (story_json, item) => {
|
|
const media_img = story_json.ledeImageUrl || Object.values(story_json.imageAttachments ?? {})[0]?.url;
|
|
const rss_item = {
|
|
title: story_json.headline || item.title,
|
|
link: story_json.url || item.link,
|
|
guid: `bloomberg:${story_json.id}`,
|
|
description: processHeadline(story_json) + (await processLedeMedia(story_json)) + (await documentToHtmlString(story_json.body || '')),
|
|
pubDate: parseDate(story_json.publishedAt) || item.pubDate,
|
|
author: story_json.authors?.map((a) => a.name).join(', ') ?? [],
|
|
category: story_json.mostRelevantTags ?? [],
|
|
media: {
|
|
content: { url: media_img },
|
|
thumbnails: { url: media_img },
|
|
},
|
|
};
|
|
return rss_item;
|
|
};
|
|
|
|
const processHeadline = (story_json) => {
|
|
const dek = story_json.dek || story_json.summary || story_json.headline || '';
|
|
const abs = story_json.abstract?.map((a) => `<li>${a}</li>`).join('');
|
|
return abs ? dek + `<ul>${abs}</ul>` : dek;
|
|
};
|
|
|
|
const processLedeMedia = async (story_json) => {
|
|
if (story_json.ledeMediaKind) {
|
|
const kind = story_json.ledeMediaKind;
|
|
|
|
const media = {
|
|
kind: story_json.ledeMediaKind,
|
|
caption: story_json.ledeCaption?.replaceAll(capRegex, '') ?? '',
|
|
description: story_json.ledeDescription?.replaceAll(capRegex, '') ?? '',
|
|
credit: story_json.ledeCredit?.replaceAll(capRegex, '') ?? '',
|
|
src: story_json.ledeImageUrl,
|
|
video: kind === 'video' && (await processVideo(story_json.ledeAttachment.bmmrId)),
|
|
};
|
|
return art(path.join(__dirname, 'templates/lede_media.art'), { media });
|
|
} else if (story_json.lede) {
|
|
const lede = story_json.lede;
|
|
const image = {
|
|
src: lede.url,
|
|
alt: lede.alt || lede.title,
|
|
caption: lede.caption?.replaceAll(capRegex, '') ?? '',
|
|
credit: lede.credit?.replaceAll(capRegex, '') ?? '',
|
|
};
|
|
return art(path.join(__dirname, 'templates/image_figure.art'), image);
|
|
} else if (story_json.imageAttachments) {
|
|
const attachment = Object.values(story_json.imageAttachments)[0];
|
|
if (attachment) {
|
|
const image = {
|
|
src: attachment.baseUrl || attachment.url,
|
|
alt: attachment.alt || attachment.title,
|
|
caption: attachment.caption?.replaceAll(capRegex, '') ?? '',
|
|
credit: attachment.credit?.replaceAll(capRegex, '') ?? '',
|
|
};
|
|
return art(path.join(__dirname, 'templates/image_figure.art'), image);
|
|
}
|
|
return '';
|
|
} else if (story_json.type === 'Lede') {
|
|
const props = story_json.props;
|
|
|
|
const media = {
|
|
kind: props.media,
|
|
caption: props.caption?.replaceAll(capRegex, '') ?? '',
|
|
description: props.dek?.replaceAll(capRegex, '') ?? '',
|
|
credit: props.credit?.replaceAll(capRegex, '') ?? '',
|
|
src: props.url,
|
|
};
|
|
return art(path.join(__dirname, 'templates/lede_media.art'), { media });
|
|
}
|
|
};
|
|
|
|
const processBody = async (body_html, story_json) => {
|
|
const removeSel = ['meta', 'script', '*[class$="-footnotes"]', '*[class$="for-you"]', '*[class$="-newsletter"]', '*[class$="page-ad"]', '*[class$="-recirc"]', '*[data-ad-placeholder="Advertisement"]'];
|
|
|
|
const $ = cheerio.load(body_html);
|
|
removeSel.forEach((sel) => $(sel).remove());
|
|
$('.paywall').removeAttr('class');
|
|
|
|
// Asynchronous iteration intentionally
|
|
// https://github.com/eslint/eslint/blob/8a159686f9d497262d573dd601855ce28362199b/tests/lib/rules/no-await-in-loop.js#L50
|
|
for await (const e of $('figure')) {
|
|
const imageType = $(e).data('image-type');
|
|
const type = $(e).data('type');
|
|
|
|
let new_figure = '';
|
|
if (imageType === 'audio') {
|
|
let audio = {};
|
|
if (story_json.audios) {
|
|
const attachment = story_json.audios.find((a) => a.id.toString() === $(e).data('id').toString());
|
|
audio = {
|
|
img: attachment.image?.url || $(e).find('img').attr('src'),
|
|
src: attachment.url || $(e).find('audio source').attr('src'),
|
|
caption: $(e).find('[class$="text"]').html()?.trim() ?? '',
|
|
credit: $(e).find('[class$="credit"]').html()?.trim() ?? '',
|
|
};
|
|
}
|
|
if (story_json.episode) {
|
|
const episode = story_json.episode;
|
|
audio = {
|
|
src: episode.url,
|
|
img: episode.image || story_json.pageInfo.image.url,
|
|
|
|
caption: episode.description || ($(e).find('[class$="text"]').html()?.trim() ?? ''),
|
|
credit: (episode.credits.map((c) => c.name).join(', ') ?? []) || ($(e).find('[class$="credit"]').html()?.trim() ?? ''),
|
|
};
|
|
}
|
|
new_figure = art(path.join(__dirname, 'templates/audio_media.art'), audio);
|
|
} else if (imageType === 'video') {
|
|
if (story_json.videoAttachments) {
|
|
const attachment = story_json.videoAttachments[$(e).data('id')];
|
|
const video = await processVideo(attachment.bmmrId);
|
|
new_figure = art(path.join(__dirname, 'templates/video_media.art'), video);
|
|
}
|
|
} else if (imageType === 'photo' || imageType === 'image' || type === 'image') {
|
|
let src, alt;
|
|
if (story_json.imageAttachments) {
|
|
const attachment = story_json.imageAttachments[$(e).data('id')];
|
|
alt = attachment?.alt || $(e).find('img').attr('alt')?.trim();
|
|
src = attachment?.baseUrl;
|
|
} else {
|
|
alt = $(e).find('img').attr('alt').trim();
|
|
src = $(e).find('img').data('native-src');
|
|
}
|
|
const caption = $(e).find('[class$="text"], .caption, .photo-essay__text').html()?.trim() ?? '';
|
|
const credit = $(e).find('[class$="credit"], .credit, .photo-essay__source').html()?.trim() ?? '';
|
|
const image = { src, alt, caption, credit };
|
|
new_figure = art(path.join(__dirname, 'templates/image_figure.art'), image);
|
|
}
|
|
$(new_figure).insertAfter(e);
|
|
$(e).remove();
|
|
}
|
|
|
|
return $.html();
|
|
};
|
|
|
|
const processVideo = async (bmmrId, summary) => {
|
|
const api = `https://www.bloomberg.com/multimedia/api/embed?id=${bmmrId}`;
|
|
const res = await got(api, { headers });
|
|
|
|
// Blocked by PX3, return the default
|
|
const redirectUrls = res.redirectUrls.map(String);
|
|
if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html')) {
|
|
return {
|
|
stream: '',
|
|
mp4: '',
|
|
coverUrl: '',
|
|
caption: summary,
|
|
};
|
|
}
|
|
|
|
if (res.data) {
|
|
const video_json = res.data;
|
|
return {
|
|
stream: video_json.streams ? video_json.streams[0]?.url : '',
|
|
mp4: video_json.downloadURLs ? video_json.downloadURLs['600'] : '',
|
|
coverUrl: video_json.thumbnail?.baseUrl ?? '',
|
|
caption: video_json.description || video_json.title || summary,
|
|
};
|
|
}
|
|
return {};
|
|
};
|
|
|
|
const nodeRenderers = {
|
|
paragraph: async (node, nextNode) => `<p>${await nextNode(node.content)}</p>`,
|
|
text: (node) => {
|
|
const { attributes: attr, value: val } = node;
|
|
if (attr?.emphasis && attr?.strong) {
|
|
return `<strong><em>${val}</em></strong>`;
|
|
} else if (attr?.emphasis) {
|
|
return `<em>${val}</em>`;
|
|
} else if (attr?.strong) {
|
|
return `<strong>${val}</strong>`;
|
|
} else {
|
|
return val;
|
|
}
|
|
},
|
|
'inline-newsletter': async (node, nextNode) => `<div>${await nextNode(node.content)}</div>`,
|
|
'inline-recirc': async (node, nextNode) => `<div>${await nextNode(node.content)}</div>`,
|
|
heading: async (node, nextNode) => {
|
|
const nodeData = node.data;
|
|
if (nodeData.level === 2 || nodeData.level === 3) {
|
|
return `<h3>${await nextNode(node.content)}</h3>`;
|
|
}
|
|
},
|
|
link: async (node, nextNode) => {
|
|
const dest = node.data.destination;
|
|
const web = dest.web;
|
|
const bbg = dest.bbg;
|
|
const title = node.data.title;
|
|
if (web) {
|
|
return `<a href="${web}" title="${title}" target="_blank">${await nextNode(node.content)}</a>`;
|
|
}
|
|
|
|
if (bbg && bbg.startsWith('bbg://news/stories')) {
|
|
const o = bbg.split('bbg://news/stories/').pop();
|
|
const s = 'https://www.bloomberg.com/news/terminal/'.concat(o);
|
|
return `<a href="${s}" title="${title}" target="_blank">${await nextNode(node.content)}</a>`;
|
|
}
|
|
return String(await nextNode(node.content));
|
|
},
|
|
entity: async (node, nextNode) => {
|
|
const t = node.subType;
|
|
const linkDest = node.data.link.destination;
|
|
const web = linkDest.web;
|
|
if (t === 'person') {
|
|
return nextNode(node.content);
|
|
}
|
|
if (t === 'story') {
|
|
if (web) {
|
|
return `<a href="${web}" target="_blank">${await nextNode(node.content)}</a>`;
|
|
}
|
|
const a = node.data.story.identifiers.suid;
|
|
const o = 'https://www.bloomberg.com/news/terminal/'.concat(a);
|
|
return `<a href="${o}" target="_blank">${await nextNode(node.content)}</a>`;
|
|
}
|
|
if (t === 'security') {
|
|
const s = node.data.security.identifiers.parsekey;
|
|
if (s) {
|
|
const c = s.split(' ');
|
|
const href = 'https://www.bloomberg.com/quote/'.concat(c[0], ':').concat(c[1]);
|
|
return `<a href="${href}" target="_blank">${await nextNode(node.content)}</a>`;
|
|
}
|
|
}
|
|
return nextNode(node.content);
|
|
},
|
|
br: () => `<br/>`,
|
|
hr: () => `<br/>`,
|
|
ad: () => {},
|
|
blockquote: async (node, nextNode) => `<blockquote>${await nextNode(node.content)}</blockquote>`,
|
|
quote: async (node, nextNode) => `<blockquote>${await nextNode(node.content)}</blockquote>`,
|
|
aside: async (node, nextNode) => `<aside>${await nextNode(node.content)}</aside>`,
|
|
list: async (node, nextNode) => {
|
|
const t = node.subType;
|
|
if (t === 'unordered') {
|
|
return `<ul>${await nextNode(node.content)}</ul>`;
|
|
}
|
|
if (t === 'ordered') {
|
|
return `<ol>${await nextNode(node.content)}</ol>`;
|
|
}
|
|
},
|
|
listItem: async (node, nextNode) => `<li>${await nextNode(node.content)}</li>`,
|
|
media: async (node) => {
|
|
const t = node.subType;
|
|
if (t === 'chart' && node.data.attachment) {
|
|
if (node.data.attachment.creator === 'TOASTER') {
|
|
const c = node.data.chart;
|
|
const e = {
|
|
src: (c && c.fallback) || '',
|
|
chart: node.data.attachment,
|
|
id: (c && c.id) || '',
|
|
alt: (c && c.alt) || '',
|
|
};
|
|
const w = e.chart;
|
|
|
|
const chart = {
|
|
source: w.source,
|
|
footnote: w.footnote,
|
|
url: w.url,
|
|
title: w.title,
|
|
subtitle: w.subtitle,
|
|
chartId: 'toaster-chart-'.concat(e.id),
|
|
chartAlt: e.alt,
|
|
fallback: e.src,
|
|
};
|
|
return art(path.join(__dirname, 'templates/chart_media.art'), { chart });
|
|
}
|
|
const image = {
|
|
alt: node.data.attachment?.footnote || '',
|
|
caption: node.data.attachment?.title + node.data.attachment.subtitle || '',
|
|
credit: node.data.attachment?.source || '',
|
|
src: node.data.chart?.fallback || '',
|
|
};
|
|
return art(path.join(__dirname, 'templates/image_figure.art'), image);
|
|
}
|
|
if (t === 'photo') {
|
|
const h = node.data;
|
|
let img = '';
|
|
if (h.attachment) {
|
|
const image = { src: h.photo?.src, alt: h.photo?.alt, caption: h.photo?.caption, credit: h.photo?.credit };
|
|
img = art(path.join(__dirname, 'templates/image_figure.art'), image);
|
|
}
|
|
if (h.link && h.link.destination && h.link.destination.web) {
|
|
const href = h.link.destination.web;
|
|
return `<a href="${href}" target="_blank">${img}</a>`;
|
|
}
|
|
return img;
|
|
}
|
|
if (t === 'video') {
|
|
const h = node.data;
|
|
const id = h.attachment?.id;
|
|
if (id) {
|
|
const desc = await processVideo(id, h.attachment?.title);
|
|
return art(path.join(__dirname, 'templates/video_media.art'), desc);
|
|
}
|
|
}
|
|
if (t === 'audio' && node.data.attachment) {
|
|
const B = node.data.attachment;
|
|
const P = B.title;
|
|
const D = B.url;
|
|
const M = B.image;
|
|
if (P && D) {
|
|
const audio = {
|
|
src: D,
|
|
img: M.url,
|
|
caption: P,
|
|
credit: '',
|
|
};
|
|
return art(path.join(__dirname, 'templates/audio_media.art'), audio);
|
|
}
|
|
}
|
|
return '';
|
|
},
|
|
tabularData: async (node, nextNode) => `<table>${await nextNode(node.content)}</table>`,
|
|
columns: (node) => {
|
|
const cols = node.data.definitions
|
|
.map((e) => ({
|
|
title: e.title,
|
|
span: e.colSpan || 1,
|
|
type: e.dataType,
|
|
}))
|
|
.map((e) => `<th colspan=${e.span}>${e.title}</th>`);
|
|
return `<tr>${cols}</tr>`;
|
|
},
|
|
row: async (node, nextNode) => `<tr>${await nextNode(node.content)}</tr>`,
|
|
cell: async (node, nextNode) => {
|
|
const types = { 'news-rsf-table-number': 'number', 'news-rsf-table-string': 'text' };
|
|
const cellType = types[node.data.class] || 'text';
|
|
return `<td data-coltype=${cellType} colspan=${node.data.colspan}>${await nextNode(node.content)}</td>`;
|
|
},
|
|
};
|
|
|
|
const nodeToHtmlString = async (node, obj) => {
|
|
const nextNode = async (nodes) => {
|
|
const nodeStr = await nodeListToHtmlString(nodes);
|
|
return nodeStr;
|
|
};
|
|
if (!node.type || !nodeRenderers[node.type]) {
|
|
return `<node>${node.type}</node>`;
|
|
}
|
|
const str = await nodeRenderers[node.type](node, nextNode, obj);
|
|
return str;
|
|
};
|
|
|
|
const nodeListToHtmlString = async (nodes) => {
|
|
const res = await Promise.all(
|
|
nodes.map(async (node, index) => {
|
|
const str = await nodeToHtmlString(node, {
|
|
index,
|
|
prev: nodes[index - 1]?.type,
|
|
next: nodes[index + 1]?.type,
|
|
});
|
|
return str;
|
|
})
|
|
);
|
|
return res.join('');
|
|
};
|
|
|
|
const documentToHtmlString = async (document) => {
|
|
if (!document || !document.content) {
|
|
return '';
|
|
}
|
|
const str = await nodeListToHtmlString(document.content);
|
|
return str;
|
|
};
|
|
|
|
const asyncPoolAll = async (...args) => {
|
|
const results = [];
|
|
for await (const result of asyncPool(...args)) {
|
|
results.push(result);
|
|
}
|
|
return results;
|
|
};
|
|
|
|
module.exports = {
|
|
rootUrl,
|
|
asyncPoolAll,
|
|
parseNewsList,
|
|
parseArticle,
|
|
};
|