diff --git a/lib/routes/bloomberg/index.ts b/lib/routes/bloomberg/index.ts index b54ea9296..c341186c4 100644 --- a/lib/routes/bloomberg/index.ts +++ b/lib/routes/bloomberg/index.ts @@ -2,29 +2,37 @@ import pMap from 'p-map'; import type { Route } from '@/types'; import { ViewType } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; +import rssParser from '@/utils/rss-parser'; -import { parseArticle, parseNewsList, rootUrl } from './utils'; +import { parseArticle, rootUrl } from './utils'; const siteTitleMapping = { '/': 'News', - bpol: 'Politics', - bbiz: 'Business', + politics: 'Politics', + business: 'Business', markets: 'Markets', technology: 'Technology', - green: 'Green', wealth: 'Wealth', - pursuits: 'Pursuits', bview: 'Opinion', - equality: 'Equality', businessweek: 'Businessweek', - citylab: 'CityLab', + economics: 'Economics', + industries: 'Industries', + crypto: 'Crypto', +}; + +// Bloomberg removed the per-site news sitemaps; map legacy site IDs to the equivalent RSS feeds +const legacySiteMapping = { + bpol: 'politics', + bbiz: 'business', }; export const route: Route = { path: '/:site?', categories: ['finance'], view: ViewType.Articles, - example: '/bloomberg/bbiz', + example: '/bloomberg/business', parameters: { site: { description: 'Site ID, can be found below', @@ -44,29 +52,37 @@ export const route: Route = { description: `| Site ID | Title | | ------------ | ------------ | | / | News | -| bpol | Politics | -| bbiz | Business | +| politics | Politics | +| business | Business | | markets | Markets | | technology | Technology | -| green | Green | | wealth | Wealth | -| pursuits | Pursuits | | bview | Opinion | -| equality | Equality | | businessweek | Businessweek | -| citylab | CityLab |`, +| economics | Economics | +| industries | Industries | +| crypto | Crypto | + +Legacy site IDs \`bpol\` and \`bbiz\` still work as aliases of \`politics\` and \`business\`.`, handler, }; async function handler(ctx) { const site = ctx.req.param('site'); - const currentUrl = site ? `${rootUrl}/${site}/sitemap_news.xml` : `${rootUrl}/sitemap_news.xml`; + const mappedSite = site ? (legacySiteMapping[site] ?? site) : undefined; + const currentUrl = mappedSite ? `${rootUrl}/${mappedSite}/news.rss` : `${rootUrl}/news.rss`; - const list = await parseNewsList(currentUrl, ctx); + const feed = await rssParser.parseString(await ofetch(currentUrl)); + const list = feed.items.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 50).map((item) => ({ + title: item.title, + link: item.link, + pubDate: item.pubDate ? parseDate(item.pubDate) : undefined, + description: item.content, + })); const items = await pMap(list, (item) => parseArticle(item), { concurrency: 1 }); return { - title: `Bloomberg - ${siteTitleMapping[site ?? '/']}`, - link: currentUrl, + title: `Bloomberg - ${siteTitleMapping[mappedSite ?? '/'] ?? feed.title}`, + link: feed.link ?? currentUrl, item: items, }; } diff --git a/lib/routes/bloomberg/utils.ts b/lib/routes/bloomberg/utils.ts index 3237d0bf6..34e5476a4 100644 --- a/lib/routes/bloomberg/utils.ts +++ b/lib/routes/bloomberg/utils.ts @@ -2,7 +2,6 @@ import { load } from 'cheerio'; import { destr } from 'destr'; import cache from '@/utils/cache'; -import got from '@/utils/got'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; @@ -71,28 +70,6 @@ const redirectGot = (url) => }), }); -const parseNewsList = async (url, ctx) => { - const resp = await got(url); - const $ = load(resp.data, { - xml: { - xmlMode: true, - }, - }); - const urls = $('urlset url'); - return urls - .toArray() - .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 50) - .map((u) => { - u = $(u); - const item = { - title: u.find(String.raw`news\:title`).text(), - link: u.find('loc').text(), - pubDate: parseDate(u.find(String.raw`news\:publication_date`).text()), - }; - return item; - }); -}; - const parseArticle = (item) => cache.tryGet(item.link, async () => { const group = regex @@ -119,6 +96,7 @@ const parseArticle = (item) => title: item.title, link: item.link, pubDate: item.pubDate, + description: item.description, }; } } @@ -130,6 +108,7 @@ const parseArticle = (item) => title: item.title, link: item.link, pubDate: item.pubDate, + description: item.description, }; } @@ -611,4 +590,4 @@ const documentToHtmlString = async (document) => { return str; }; -export { parseArticle, parseNewsList, rootUrl }; +export { parseArticle, rootUrl };