feat(route): add KCNA (#9792)

Signed-off-by: Rongrong <i@rong.moe>
2022-05-21 03:34:09 +08:00 · 2022-05-21 03:34:09 +08:00 · f3de5e39bc
parent 1f84316387
commit f3de5e39bc
8 changed files with 269 additions and 0 deletions
--- a/docs/en/traditional-media.md
+++ b/docs/en/traditional-media.md
@ -175,6 +175,30 @@ Generates full-text feeds that the official feed doesn't provide.

 </RouteEn>

+## Korean Central News Agency (KCNA)
+
+### News
+
+<RouteEn author="Rongronggg9" example="/kcna/en" path="/kcna/:lang/:category?" :paramsDesc="['Language, refer to the table below', 'Category, refer to the table below']" anticrawler="1" radar="1" rssbud="1">
+
+| Language | 조선어  | English | 中国语  | Русский | Español | 日本語  |
+|----------|------|---------|------|---------|---------|------|
+| `:lang`  | `kp` | `en`    | `cn` | `ru`    | `es`    | `jp` |
+
+| Category                                                         | `:category`                        |
+|------------------------------------------------------------------|------------------------------------|
+| WPK General Secretary **Kim Jong Un**'s Revolutionary Activities | `54c0ca4ca013a92cc9cf95bd4004c61a` |
+| Latest News (default)                                            | `1ee9bdb7186944f765208f34ecfb5407` |
+| Top News                                                         | `5394b80bdae203fadef02522cfb578c0` |
+| Home News                                                        | `b2b3bcc1b0a4406ab0c36e45d5db58db` |
+| Documents                                                        | `a8754921399857ebdbb97a98a1e741f5` |
+| World                                                            | `593143484cf15d48ce85c26139582395` |
+| Society-Life                                                     | `93102e5a735d03979bc58a3a7aefb75a` |
+| External                                                         | `0f98b4623a3ef82aeea78df45c423fd0` |
+| News Commentary                                                  | `12c03a49f7dbe829bceea8ac77088c21` |
+
+</RouteEn>
+
 ## Ming Pao

 ### Ming Pao Daily
--- a/docs/traditional-media.md
+++ b/docs/traditional-media.md
@ -670,6 +670,30 @@ IT・科学 tech_science

 </Route>

+## 朝鲜中央通讯社
+
+### 新闻
+
+<Route author="Rongronggg9" example="/kcna/cn" path="/kcna/:lang/:category?" :paramsDesc="['语言，见下表', '分类，见下表']" anticrawler="1" radar="1" rssbud="1">
+
+| 语言      | 조선어  | English | 中国语  | Русский | Español | 日本語  |
+| ------- | ---- | ------- | ---- | ------- | ------- | ---- |
+| `:lang` | `kp` | `en`    | `cn` | `ru`    | `es`    | `jp` |
+
+| 分类                      | `:category`                        |
+| ----------------------- | ---------------------------------- |
+| 朝鲜劳动党总书记**金正恩**同志革命活动新闻 | `54c0ca4ca013a92cc9cf95bd4004c61a` |
+| 最新新闻 (默认)               | `1ee9bdb7186944f765208f34ecfb5407` |
+| 主要新闻                    | `5394b80bdae203fadef02522cfb578c0` |
+| 国内新闻                    | `b2b3bcc1b0a4406ab0c36e45d5db58db` |
+| 文件                      | `a8754921399857ebdbb97a98a1e741f5` |
+| 国际新闻                    | `593143484cf15d48ce85c26139582395` |
+| 社会－生活                   | `93102e5a735d03979bc58a3a7aefb75a` |
+| 对外关系                    | `0f98b4623a3ef82aeea78df45c423fd0` |
+| 时事解说                    | `12c03a49f7dbe829bceea8ac77088c21` |
+
+</Route>
+
 ## 第一财经

 ### 直播区
--- a/lib/v2/kcna/maintainer.js
+++ b/lib/v2/kcna/maintainer.js
@ -0,0 +1,3 @@
+module.exports = {
+    '/:lang/:category?': ['Rongronggg9'],
+};
--- a/lib/v2/kcna/news.js
+++ b/lib/v2/kcna/news.js
@ -0,0 +1,81 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const asyncPool = require('tiny-async-pool');
+const { art } = require('@/utils/render');
+const { parseJucheDate, fixDesc, fetchPhoto, fetchVideo } = require('./utils');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const { lang, category = '1ee9bdb7186944f765208f34ecfb5407' } = ctx.params;
+
+    const rootUrl = 'http://www.kcna.kp';
+    const pageUrl = `${rootUrl}/${lang}/category/articles/q/${category}.kcmsf`;
+
+    const response = await got(pageUrl);
+    const $ = cheerio.load(response.data);
+
+    // fix <nobr><span class="fSpecCs">???</span></nobr>
+    const title = $('head > title')
+        .text()
+        .replace(/<[^>]*>/g, '');
+
+    const list = $('.article-link li a')
+        .map((_, item) => {
+            item = $(item);
+            const dateElem = item.find('.publish-time');
+            const dateString = dateElem.text();
+            dateElem.remove();
+            return {
+                title: item.text(),
+                link: rootUrl + item.attr('href'),
+                pubDate: parseJucheDate(dateString),
+            };
+        })
+        .get();
+
+    // avoid being IP-banned
+    // if being banned, 103.35.255.254 (the last hop before www.kcna.kp - 175.45.176.71) will drop the packet
+    // verify that with `mtr www.kcna.kp -Tz`
+    const items = [];
+    for await (const item of asyncPool(3, list, (item) =>
+        ctx.cache.tryGet(item.link, async () => {
+            const response = await got(item.link);
+            const $ = cheerio.load(response.data);
+            item.title = $('article-main-title').text() || item.title;
+
+            const dateElem = $('.publish-time');
+            const dateString = dateElem.text();
+            dateElem.remove();
+            item.pubDate = parseJucheDate(dateString) || item.pubDate;
+
+            const description = fixDesc($, $('.article-content-body .content-wrapper'));
+
+            // add picture and video
+            const media = $('.media-icon a')
+                .map((_, elem) => rootUrl + elem.attribs.href)
+                .get();
+            let photo, video;
+            await Promise.all(
+                media.map(async (medium) => {
+                    if (medium.includes('/photo/')) {
+                        photo = await fetchPhoto(ctx, medium);
+                    } else if (medium.includes('/video/')) {
+                        video = await fetchVideo(ctx, medium);
+                    }
+                })
+            );
+
+            item.description = art(path.join(__dirname, 'templates/news.art'), { description, photo, video });
+
+            return item;
+        })
+    )) {
+        items.push(item);
+    }
+
+    ctx.state.data = {
+        title,
+        link: pageUrl,
+        item: items,
+    };
+};
--- a/lib/v2/kcna/radar.js
+++ b/lib/v2/kcna/radar.js
@ -0,0 +1,61 @@
+module.exports = {
+    'kcna.kp': {
+        _name: '朝鲜中央通讯社',
+        www: [
+            {
+                title: '朝鲜劳动党总书记金正恩同志革命活动新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#chao-xian-zhong-yang-tong-xun-she',
+                source: '/:lang/category/articles/q/54c0ca4ca013a92cc9cf95bd4004c61a.kcmsf',
+                target: '/kcna/:lang/54c0ca4ca013a92cc9cf95bd4004c61a',
+            },
+            {
+                title: '最新新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#chao-xian-zhong-yang-tong-xun-she',
+                source: ['/:lang', '/:lang/category/articles/q/1ee9bdb7186944f765208f34ecfb5407.kcmsf', '/:lang/category/articles.kcmsf'],
+                target: '/kcna/:lang',
+            },
+            {
+                title: '主要新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#chao-xian-zhong-yang-tong-xun-she',
+                source: '/:lang/category/articles/q/5394b80bdae203fadef02522cfb578c0.kcmsf',
+                target: '/kcna/:lang/5394b80bdae203fadef02522cfb578c0',
+            },
+            {
+                title: '国内新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#chao-xian-zhong-yang-tong-xun-she',
+                source: '/:lang/category/articles/q/b2b3bcc1b0a4406ab0c36e45d5db58db.kcmsf',
+                target: '/kcna/:lang/b2b3bcc1b0a4406ab0c36e45d5db58db',
+            },
+            {
+                title: '文件',
+                docs: 'https://docs.rsshub.app/traditional-media.html#chao-xian-zhong-yang-tong-xun-she',
+                source: '/:lang/category/articles/q/a8754921399857ebdbb97a98a1e741f5.kcmsf',
+                target: '/kcna/:lang/a8754921399857ebdbb97a98a1e741f5',
+            },
+            {
+                title: '国际新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#chao-xian-zhong-yang-tong-xun-she',
+                source: '/:lang/category/articles/q/593143484cf15d48ce85c26139582395.kcmsf',
+                target: '/kcna/:lang/593143484cf15d48ce85c26139582395',
+            },
+            {
+                title: '社会－生活',
+                docs: 'https://docs.rsshub.app/traditional-media.html#chao-xian-zhong-yang-tong-xun-she',
+                source: '/:lang/category/articles/q/93102e5a735d03979bc58a3a7aefb75a.kcmsf',
+                target: '/kcna/:lang/93102e5a735d03979bc58a3a7aefb75a',
+            },
+            {
+                title: '对外关系',
+                docs: 'https://docs.rsshub.app/traditional-media.html#chao-xian-zhong-yang-tong-xun-she',
+                source: '/:lang/category/articles/q/0f98b4623a3ef82aeea78df45c423fd0.kcmsf',
+                target: '/kcna/:lang/0f98b4623a3ef82aeea78df45c423fd0',
+            },
+            {
+                title: '时事解说',
+                docs: 'https://docs.rsshub.app/traditional-media.html#chao-xian-zhong-yang-tong-xun-she',
+                source: '/:lang/category/articles/q/12c03a49f7dbe829bceea8ac77088c21.kcmsf',
+                target: '/kcna/:lang/12c03a49f7dbe829bceea8ac77088c21',
+            },
+        ],
+    },
+};
--- a/lib/v2/kcna/router.js
+++ b/lib/v2/kcna/router.js
@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:lang/:category?', require('./news'));
+};
--- a/lib/v2/kcna/templates/news.art
+++ b/lib/v2/kcna/templates/news.art
@ -0,0 +1,9 @@
+{{@ description }}
+{{ if photo }}
+<br>
+{{@ photo }}
+{{ /if }}
+{{ if video }}
+<br>
+{{@ video }}
+{{ /if }}
--- a/lib/v2/kcna/utils.js
+++ b/lib/v2/kcna/utils.js
@ -0,0 +1,64 @@
+const { parseDate } = require('@/utils/parse-date');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const rootUrl = 'http://www.kcna.kp';
+
+const parseJucheDate = (dateString) => {
+    if (!dateString) {
+        return null;
+    }
+
+    // https://en.wikipedia.org/wiki/Juche_calendar
+    const dateMatch = dateString.match(/(\d+)\D(\d+)\D(\d+)/);
+    const [jucheYear, month, day] = dateMatch ? dateMatch.slice(1) : [null, null, null];
+    if (jucheYear && month && day) {
+        const year = parseInt(jucheYear, 10) + 1911;
+        return parseDate(`${year}-${month}-${day}`, 'YYYY-M-D');
+    }
+    return null;
+};
+
+const fixDesc = ($, elem) => {
+    // <nobr><span className='fSpecCs'>???</span></nobr> => <b>???</b>
+    const $elem = $(elem);
+    $elem.find('.fSpecCs').each((_, item) => {
+        if (item.parent.name === 'nobr') {
+            $(item).unwrap();
+        }
+        item.name = 'b';
+        item.attribs = {};
+    });
+    return elem.html();
+};
+
+const fetchPhoto = (ctx, url) =>
+    ctx.cache.tryGet(url, async () => {
+        const res = await got(url);
+        const $ = cheerio.load(res.data);
+        let html = '';
+        $('.content img').each((_, item) => {
+            const src = item.attribs.src;
+            if (src) {
+                html += html ? `<br><img src="${src}">` : `<img src="${src}">`;
+            }
+        });
+        return html;
+    });
+
+const fetchVideo = (ctx, url) =>
+    ctx.cache.tryGet(url, async () => {
+        const res = await got(url);
+        const $ = cheerio.load(res.data);
+        const js = $('script[type="text/javascript"]:not([src])').html();
+        let sources = js.match(/<[^>]*source[^>]+src[^>]+>/g);
+        sources = sources && sources.map((item) => item.replace(/'/g, '"').replace(/src="([^"]+)"/g, `src="${rootUrl}$1"`));
+        return `<video controls preload="none">${sources.join('\n')}</video>`;
+    });
+
+module.exports = {
+    parseJucheDate,
+    fixDesc,
+    fetchPhoto,
+    fetchVideo,
+};