From 633aec3dacdbce30ec832fe30edd1dbedca9b11a Mon Sep 17 00:00:00 2001
From: Evan <yif.wang@icloud.com>
Date: Mon, 13 Mar 2023 07:38:44 -0700
Subject: [PATCH] refactor(route): wsj news (#12087)

* feat(route): wsj news

* chore: docs update

* chore: docs update

* chore: use map to rewrite the function

* chore: update utils

* Update docs/en/traditional-media.md
---------
---
 docs/en/traditional-media.md |  12 ++-
 docs/traditional-media.md    |  14 +++-
 lib/v2/wsj/index.js          | 141 -----------------------------------
 lib/v2/wsj/news.js           |  51 +++++++++++++
 lib/v2/wsj/radar.js          |   4 +-
 lib/v2/wsj/router.js         |   2 +-
 lib/v2/wsj/utils.js          |  99 ++++++++++++++++++++++++
 7 files changed, 177 insertions(+), 146 deletions(-)
 delete mode 100644 lib/v2/wsj/index.js
 create mode 100644 lib/v2/wsj/news.js
 create mode 100644 lib/v2/wsj/utils.js

diff --git a/docs/en/traditional-media.md b/docs/en/traditional-media.md
index 00685cf76..0acd79778 100644
--- a/docs/en/traditional-media.md
+++ b/docs/en/traditional-media.md
@@ -678,7 +678,17 @@ Provides all of the articles by the specified New York Times author.
 
 ### News
 
-<RouteEn author="oppilate" example="/wsj/en-us/opinion" path="/wsj/:lang/:category?" :paramsDesc="['Language, `en-us`, `zh-cn`, `zh-tw` are supported', 'Category, only supported in `en-us`. Supports `opinion`, `world_news`, `us_bussiness`, `market_news`, `technology`, `lifestyle`.']">
+<RouteEn author="oppilate NavePnow" example="/wsj/en-us/opinion" path="/wsj/:lang/:category?" :paramsDesc="['Language, `en-us`, `zh-cn`, `zh-tw`', 'Category. See below']">
+
+en_us
+| World | U.S. | Politics | Economy | Business | Tech | Markets | Opinion | Books & Arts | Real Estate | Life & Work | Sytle | Sports |
+| ------ | ------- | -------- | -------- | ----- | --------- | --------- | --------- | --------- | --------- |--------- | --------- | --------- |
+| world | us | politics | economy | business | technology | markets | opinion | books-arts | realestate | life-work | style-entertainment | sports |
+
+zh-cn / zh-tw
+| 国际 | 中国 | 金融市场 | 经济 | 商业 | 科技 | 派 | 专栏与观点 |
+| ------ | ------- | -------- | -------- | ----- | --------- | --------- | --------- |
+| world | china | markets | economy | business | technology | life-arts | opinion |
 
 Provide full article RSS for WSJ topics.
 
diff --git a/docs/traditional-media.md b/docs/traditional-media.md
index d80084716..cdb5bb6e1 100644
--- a/docs/traditional-media.md
+++ b/docs/traditional-media.md
@@ -1182,7 +1182,19 @@ IT・科学 tech_science
 
 ### 新闻
 
-<Route author="oppilate" example="/wsj/en-us/opinion" path="/wsj/:lang/:category?" :paramsDesc="['语言，支持 `en-us`、`zh-cn`、`zh-tw`', '分类，仅 `en-us` 支持分类订阅。支持 `opinion`, `world_news`, `us_bussiness`, `market_news`, `technology`, `lifestyle`。']">
+<Route author="oppilate NavePnow" example="/wsj/en-us/opinion" path="/wsj/:lang/:category?" :paramsDesc="['语言，支持 `en-us`、`zh-cn`、`zh-tw`', '分类，见下表']">
+
+en_us
+
+| World | U.S. | Politics | Economy | Business | Tech       | Markets | Opinion | Books & Arts | Real Estate | Life & Work | Sytle               | Sports |
+| ----- | ---- | -------- | ------- | -------- | ---------- | ------- | ------- | ------------ | ----------- | ----------- | ------------------- | ------ |
+| world | us   | politics | economy | business | technology | markets | opinion | books-arts   | realestate  | life-work   | style-entertainment | sports |
+
+zh-cn / zh-tw
+
+| 国际    | 中国    | 金融市场    | 经济      | 商业       | 科技         | 派         | 专栏与观点   |
+| ----- | ----- | ------- | ------- | -------- | ---------- | --------- | ------- |
+| world | china | markets | economy | business | technology | life-arts | opinion |
 
 通过提取文章全文，以提供比官方源更佳的阅读体验。
 
diff --git a/lib/v2/wsj/index.js b/lib/v2/wsj/index.js
deleted file mode 100644
index 4c7b82c0c..000000000
--- a/lib/v2/wsj/index.js
+++ /dev/null
@@ -1,141 +0,0 @@
-const parser = require('@/utils/rss-parser');
-const cheerio = require('cheerio');
-const got = require('@/utils/got');
-const { parseDate } = require('@/utils/parse-date');
-
-const categoryToXMLFileName = {
-    opinion: 'RSSOpinion.xml',
-    world_news: 'RSSWorldNews.xml',
-    us_bussiness: 'WSJcomUSBusiness.xml',
-    market_news: 'RSSMarketsMain.xml',
-    technology: 'RSSWSJD.xml',
-    lifestyle: 'RSSLifestyle.xml',
-};
-
-const categoryToName = {
-    opinion: 'Opinion',
-    world_news: 'World News',
-    us_bussiness: 'U.S. Business',
-    market_news: 'Markets News',
-    technology: "Technology: What's News",
-    lifestyle: 'Lifestyle',
-};
-
-module.exports = async (ctx) => {
-    const language = ctx.params.lang;
-    const category = ctx.params.category;
-    let rssUrl;
-    switch (language) {
-        case 'en-us':
-            rssUrl = `https://feeds.a.dj.com/rss/${categoryToXMLFileName[category]}`;
-            break;
-        case 'zh-cn':
-            // Doesn't support categorical subscribtion
-            rssUrl = `https://cn.wsj.com/zh-hans/rss/`;
-            break;
-        case 'zh-tw':
-            rssUrl = `https://cn.wsj.com/zh-hant/rss/`;
-            break;
-        default:
-            // Doesn't support other languages (e.g. ja) for now
-            throw Error(`Language ${language} is not supported`);
-    }
-
-    const feed = await parser.parseURL(rssUrl);
-    const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
-
-    const items = await Promise.all(
-        feed.items.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                // Fetch the AMP version
-                const url = item.link.replace(/(?<=^https:\/\/\w+\.wsj\.com)/, '/amp');
-                const response = await got({
-                    url,
-                    method: 'get',
-                    headers: {
-                        'User-Agent': chromeMobileUserAgent,
-                    },
-                });
-                const html = response.body;
-                const $ = cheerio.load(html);
-                const content = $('.articleBody > section');
-
-                // Cover
-                const cover = $('.articleLead > div.is-lead-inset > div.header > .img-header > div.image-container > amp-img > img');
-
-                if (cover.length > 0) {
-                    $(`<img src=${cover[0].attribs.content}>`).insertBefore(content[0].childNodes[0]);
-                    $(cover).remove();
-                }
-
-                // Summary
-                const summary = $('head > meta[name="description"]').attr('content');
-
-                // Metadata (categories & updatedAt)
-                const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
-                const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
-                const author = $('.author > a[rel="author"]').html();
-
-                const categories = $('meta[name="keywords"]')
-                    .attr('content')
-                    .split(',')
-                    .map((c) => c.trim());
-
-                // Images
-                content.find('amp-img').each((i, e) => {
-                    const img = $(`<img width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}" alt="${e.attribs.alt}">`);
-
-                    // Caption follows, no need to handle caption
-                    $(img).insertBefore(e);
-                    $(e).remove();
-                });
-
-                // iframes (youtube videos and interactive elements)
-                content.find('amp-iframe').each((i, e) => {
-                    const iframe = $(`<iframe width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}">`);
-                    $(iframe).insertBefore(e);
-                    $(e).remove();
-                });
-
-                // Remove unwanted DOMs
-                const unwanted_element_selectors = ['amp-ad', '.wsj-ad'];
-                unwanted_element_selectors.forEach((selector) => {
-                    content.find(selector).each((i, e) => {
-                        $(e).remove();
-                    });
-                });
-
-                // Paywall
-                content.find('.paywall').each((i, e) => {
-                    // Caption follows, no need to handle caption
-                    $(e.childNodes).insertBefore(e);
-                    $(e).remove();
-                });
-
-                return {
-                    title: item.title,
-                    id: item.guid,
-                    pubDate: parseDate(publishedAt),
-                    updated: parseDate(updatedAt),
-                    author,
-                    link: item.link,
-                    summary,
-                    description: content.html(),
-                    category: categories,
-                    icon: 'https://s.wsj.net/media/wsj_launcher-icon-4x.png',
-                    logo: 'https://vir.wsj.net/fp/assets/webpack4/img/wsj-logo-big-black.165e51cc.svg',
-                };
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: categoryToName[category],
-        link: feed.link,
-        description: feed.description,
-        item: items,
-        language: feed.language,
-        icon: 'https://s.wsj.net/media/wsj_launcher-icon-4x.png',
-        logo: 'https://vir.wsj.net/fp/assets/webpack4/img/wsj-logo-big-black.165e51cc.svg',
-    };
-};
diff --git a/lib/v2/wsj/news.js b/lib/v2/wsj/news.js
new file mode 100644
index 000000000..2f06c536f
--- /dev/null
+++ b/lib/v2/wsj/news.js
@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { asyncPoolAll, parseArticle } = require('./utils');
+const hostMap = {
+    'en-us': 'https://www.wsj.com',
+    'zh-cn': 'https://cn.wsj.com/zh-hans',
+    'zh-tw': 'https://cn.wsj.com/zh-hant',
+};
+module.exports = async (ctx) => {
+    const lang = ctx.params.lang;
+    const category = ctx.params.category || '';
+    const host = hostMap[lang];
+    let subTitle = ` - ${lang.toUpperCase()}`;
+    let url = host;
+    if (category.length > 0) {
+        url = `${host}/news/${category}`;
+        subTitle = `${subTitle} - ${category}`;
+    }
+    const response = await got({
+        method: 'get',
+        url,
+    });
+
+    const $ = cheerio.load(response.data);
+    const contents = $('script:contains("window.__STATE__")').text();
+    const data = JSON.parse(contents.match(/\{.*\}/)[0]).data;
+    const filteredKeys = Object.entries(data)
+        .filter(([key, value]) => {
+            if (!key.startsWith('article')) {
+                return false;
+            }
+            const link = value.data.data.url;
+            return link.includes('wsj.com/articles/');
+        })
+        .map(([key]) => key);
+    const list = filteredKeys.map((key) => {
+        const item = {};
+        item.title = data[key].data.data.headline;
+        item.link = data[key].data.data.url;
+        item.test = key;
+        return item;
+    });
+    const items = await asyncPoolAll(1, list, (item) => parseArticle(item, ctx));
+
+    ctx.state.data = {
+        title: `WSJ${subTitle}`,
+        link: url,
+        description: `WSJ${subTitle}`,
+        item: items,
+    };
+};
diff --git a/lib/v2/wsj/radar.js b/lib/v2/wsj/radar.js
index 1b59052da..909cba84a 100644
--- a/lib/v2/wsj/radar.js
+++ b/lib/v2/wsj/radar.js
@@ -5,7 +5,7 @@ module.exports = {
             {
                 title: '新闻',
                 docs: 'https://docs.rsshub.app/traditional-media.html#hua-er-jie-ri-bao-the-wall-street-journal-wsj',
-                souce: '/',
+                source: '/',
                 target: '/wsj/zh-cn',
             },
         ],
@@ -13,7 +13,7 @@ module.exports = {
             {
                 title: '新闻',
                 docs: 'https://docs.rsshub.app/traditional-media.html#hua-er-jie-ri-bao-the-wall-street-journal-wsj',
-                souce: '/',
+                source: '/',
                 target: '/wsj/en-us',
             },
         ],
diff --git a/lib/v2/wsj/router.js b/lib/v2/wsj/router.js
index accf74aab..e313ad565 100644
--- a/lib/v2/wsj/router.js
+++ b/lib/v2/wsj/router.js
@@ -1,3 +1,3 @@
 module.exports = (router) => {
-    router.get('/:lang/:category?', require('./index'));
+    router.get('/:lang/:category?', require('./news'));
 };
diff --git a/lib/v2/wsj/utils.js b/lib/v2/wsj/utils.js
new file mode 100644
index 000000000..f1a31aef2
--- /dev/null
+++ b/lib/v2/wsj/utils.js
@@ -0,0 +1,99 @@
+const asyncPool = require('tiny-async-pool');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const UA = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' });
+
+// const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
+const parseArticle = (item, ctx) =>
+    ctx.cache.tryGet(item.link, async () => {
+        // Fetch the AMP version
+        const url = item.link.replace(/(?<=^https:\/\/\w+\.wsj\.com)/, '/amp');
+        const response = await got({
+            url,
+            method: 'get',
+            headers: {
+                'User-Agent': UA,
+            },
+        });
+        const html = response.data;
+        const $ = cheerio.load(html);
+        const content = $('.articleBody > section');
+
+        // Cover
+        const cover = $('.articleLead > div.is-lead-inset > div.header > .img-header > div.image-container > amp-img > img');
+
+        if (cover.length > 0) {
+            $(`<img src=${cover[0].attribs.content}>`).insertBefore(content[0].childNodes[0]);
+            $(cover).remove();
+        }
+
+        // Summary
+        const summary = $('head > meta[name="description"]').attr('content');
+
+        // Metadata (categories & updatedAt)
+        const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
+        const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
+        const author = $('.author > a[rel="author"]').text();
+
+        const categories = $('meta[name="keywords"]')
+            .attr('content')
+            .split(',')
+            .map((c) => c.trim());
+
+        // Images
+        content.find('amp-img').each((i, e) => {
+            const img = $(`<img width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}" alt="${e.attribs.alt}">`);
+
+            // Caption follows, no need to handle caption
+            $(img).insertBefore(e);
+            $(e).remove();
+        });
+
+        // iframes (youtube videos and interactive elements)
+        content.find('amp-iframe').each((i, e) => {
+            const iframe = $(`<iframe width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}">`);
+            $(iframe).insertBefore(e);
+            $(e).remove();
+        });
+
+        // Remove unwanted DOMs
+        const unwanted_element_selectors = ['amp-ad', '.wsj-ad'];
+        unwanted_element_selectors.forEach((selector) => {
+            content.find(selector).each((i, e) => {
+                $(e).remove();
+            });
+        });
+
+        // Paywall
+        content.find('.paywall').each((i, e) => {
+            // Caption follows, no need to handle caption
+            $(e.childNodes).insertBefore(e);
+            $(e).remove();
+        });
+
+        return {
+            title: item.title,
+            pubDate: parseDate(publishedAt),
+            updated: parseDate(updatedAt),
+            author,
+            link: item.link,
+            summary,
+            description: content.html(),
+            category: categories,
+            icon: 'https://s.wsj.net/media/wsj_launcher-icon-4x.png',
+            logo: 'https://vir.wsj.net/fp/assets/webpack4/img/wsj-logo-big-black.165e51cc.svg',
+        };
+    });
+
+const asyncPoolAll = async (...args) => {
+    const results = [];
+    for await (const result of asyncPool(...args)) {
+        results.push(result);
+    }
+    return results;
+};
+module.exports = {
+    asyncPoolAll,
+    parseArticle,
+};