fix(route): bloomberg author (#11619)

* Switch bloomberg author feed to html scraper

* Safely resolve relative href

* Prefer WHATWG URL API

* Remove list cache

* Cache article list

* Avoid caching captcha

* Add back rss parser as fallback

* Load author news list from api

* Fallback to rss

* docs: add docs for new params

docs/en/finance.md

@@ -29,7 +29,7 @@ pageClass: routes
 
 ### Authors
 
-<RouteEn author="josh" example="/bloomberg/authors/ARbTQlRLRjE/matthew-s-levine" path="/bloomberg/authors/:id/:slug" :paramsDesc="['Author ID, can be found in URL', 'Author Slug, can be found in URL']" anticrawler="1" radar="1"/>
+<RouteEn author="josh" example="/bloomberg/authors/ARbTQlRLRjE/matthew-s-levine" path="/bloomberg/authors/:id/:slug/:source?" :paramsDesc="['Author ID, can be found in URL', 'Author Slug, can be found in URL', 'Data source, either `api` or `rss`,`api` by default']" anticrawler="1" radar="1"/>
 
 ## CFD
 

lib/v2/bloomberg/authors.js

@@ -1,14 +1,52 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const rssParser = require('@/utils/rss-parser');
 const { asyncPoolAll, parseArticle } = require('./utils');
-const parser = require('@/utils/rss-parser');
+
+const parseAuthorNewsList = async (slug) => {
+    const baseURL = `https://www.bloomberg.com/authors/${slug}`;
+    const apiUrl = `https://www.bloomberg.com/lineup/api/lazy_load_author_stories?slug=${slug}&authorType=default&page=1`;
+    const resp = await got(apiUrl);
+    // Likely rate limited
+    if (!resp.data.html) {
+        return [];
+    }
+    const $ = cheerio.load(resp.data.html);
+    const articles = $('article.story-list-story');
+    return articles
+        .map((index, item) => {
+            item = $(item);
+            const headline = item.find('a.story-list-story__info__headline-link');
+            return {
+                title: headline.text(),
+                pubDate: item.attr('data-updated-at'),
+                guid: `bloomberg:${item.attr('data-id')}`,
+                link: new URL(headline.attr('href'), baseURL).href,
+            };
+        })
+        .get();
+};
 
 module.exports = async (ctx) => {
-    const { id, slug } = ctx.params;
-    const feed = await parser.parseURL(`https://www.bloomberg.com/authors/${id}/${slug}.rss`);
-    const item = await asyncPoolAll(1, feed.items, (item) => parseArticle(item, ctx));
+    const { id, slug, source } = ctx.params;
+    const link = `https://www.bloomberg.com/authors/${id}/${slug}`;
+
+    let list = [];
+    if (!source || source === 'api') {
+        list = await parseAuthorNewsList(`${id}/${slug}`);
+    }
+    // Fallback to rss if api failed or requested by param
+    if (source === 'rss' || list.length === 0) {
+        list = (await rssParser.parseURL(`${link}.rss`)).items;
+    }
+
+    const item = await asyncPoolAll(1, list, (item) => parseArticle(item, ctx));
+    const authorName = item.find((i) => i.author)?.author ?? 'Unknown';
+
     ctx.state.data = {
-        title: `Bloomberg - ${feed.title.split(' - ', 2)[0]}`,
-        link: feed.link,
-        language: feed.language,
+        title: `Bloomberg - ${authorName}`,
+        link,
+        language: 'en-us',
         item,
     };
 };

lib/v2/bloomberg/maintainer.js

@@ -1,5 +1,5 @@
 module.exports = {
-    '/authors/:id/:slug': ['josh'],
+    '/authors/:id/:slug/:source?': ['josh'],
     '/:site?': ['bigfei'],
     '/': ['bigfei'],
 };

lib/v2/bloomberg/router.js

@@ -1,5 +1,5 @@
 module.exports = function (router) {
-    router.get('/authors/:id/:slug', require('./authors'));
+    router.get('/authors/:id/:slug/:source?', require('./authors'));
     router.get('/:site', require('./index'));
     router.get('/', require('./index'));
 };