feat(route/caixin): Add full text fetch (#17231)

* feat(route/caixin): Add full text fetch * . * Update blog.ts * Update utils.ts * Update utils.ts * Update utils-fulltext.ts * Update utils-fulltext.ts
2024-11-03 18:03:20 +08:00 · 2024-11-03 18:03:20 +08:00 · 35536f45ad
parent a63c8b0d60
commit 35536f45ad
8 changed files with 101 additions and 56 deletions
--- a/lib/config.ts
+++ b/lib/config.ts
@ -94,6 +94,9 @@ export type Config = {
    bupt: {
        portal_cookie?: string;
    };
+    caixin: {
+        cookie?: string;
+    };
    civitai: {
        cookie?: string;
    };
@ -500,6 +503,9 @@ const calculateValue = () => {
        bupt: {
            portal_cookie: envs.BUPT_PORTAL_COOKIE,
        },
+        caixin: {
+            cookie: envs.CAIXIN_COOKIE,
+        },
        civitai: {
            cookie: envs.CIVITAI_COOKIE,
        },
--- a/lib/routes/caixin/article.ts
+++ b/lib/routes/caixin/article.ts
@ -42,7 +42,7 @@ async function handler() {
        audio_image_url: item.audio_image_url,
    }));

-    const items = await Promise.all(list.map((item) => parseArticle(item, cache.tryGet)));
+    const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => parseArticle(item))));

    return {
        title: '财新网 - 首页',
--- a/lib/routes/caixin/blog.ts
+++ b/lib/routes/caixin/blog.ts
@ -67,8 +67,7 @@ async function handler(ctx) {
            pubDate: parseDate(item.publishTime, 'x'),
        }));

-        const items = await Promise.all(posts.map((item) => parseBlogArticle(item, cache.tryGet)));
-
+        const items = await Promise.all(posts.map((item) => cache.tryGet(item.link, () => parseBlogArticle(item))));
        return {
            title: `财新博客 - ${authorName}`,
            link,
@ -90,7 +89,7 @@ async function handler(ctx) {
            link: item.postUrl.replace('http://', 'https://'),
            pubDate: parseDate(item.publishTime, 'x'),
        }));
-        const items = await Promise.all(posts.map((item) => parseBlogArticle(item, cache.tryGet)));
+        const items = await Promise.all(posts.map((item) => cache.tryGet(item.link, () => parseBlogArticle(item))));

        return {
            title: `财新博客 - 全部`,
--- a/lib/routes/caixin/category.ts
+++ b/lib/routes/caixin/category.ts
@ -83,7 +83,7 @@ async function handler(ctx) {
        audio_image_url: item.pict.imgs[0].url,
    }));

-    const items = await Promise.all(list.map((item) => parseArticle(item, cache.tryGet)));
+    const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => parseArticle(item))));

    return {
        title,
--- a/lib/routes/caixin/latest.ts
+++ b/lib/routes/caixin/latest.ts
@ -1,12 +1,9 @@
 import { Route, ViewType } from '@/types';
-import { getCurrentPath } from '@/utils/helpers';
-const __dirname = getCurrentPath(import.meta.url);
+import { getFulltext } from './utils-fulltext';

 import cache from '@/utils/cache';
 import got from '@/utils/got';
-import { load } from 'cheerio';
-import { art } from '@/utils/render';
-import path from 'node:path';
+import { parseArticle } from './utils';

 export const route: Route = {
    path: '/latest',
@ -31,10 +28,10 @@ export const route: Route = {
    maintainers: ['tpnonthealps'],
    handler,
    url: 'caixin.com/',
-    description: `说明：此 RSS feed 会自动抓取财新网的最新文章，但不包含 FM 及视频内容。`,
+    description: `说明：此 RSS feed 会自动抓取财新网的最新文章，但不包含 FM 及视频内容。订阅用户可根据文档设置环境变量后，在url传入\`fulltext=\`以解锁全文。`,
 };

-async function handler() {
+async function handler(ctx) {
    const { data } = await got('https://gateway.caixin.com/api/dataplatform/scroll/index');

    const list = data.data.articleList
@ -49,21 +46,15 @@ async function handler() {
    const rss = await Promise.all(
        list.map((item) =>
            cache.tryGet(`caixin:latest:${item.link}`, async () => {
-                const entry_r = await got(item.link);
-                const $ = load(entry_r.data);
-
                // desc
-                const desc = art(path.join(__dirname, 'templates/article.art'), {
-                    item,
-                    $,
-                });
+                const desc = await parseArticle(item);

-                item.description = desc;
+                item.description = ctx.req.query('fulltext') === 'true' ? ((await getFulltext(item.link)) ?? desc.description) : desc.description;
                // prevent cache coliision with /caixin/article and /caixin/:column/:category
                // since those have podcasts
                item.guid = `caixin:latest:${item.link}`;

-                return item;
+                return { ...desc, ...item };
            })
        )
    );
--- a/lib/routes/caixin/namespace.ts
+++ b/lib/routes/caixin/namespace.ts
@ -3,6 +3,6 @@ import type { Namespace } from '@/types';
 export const namespace: Namespace = {
    name: '财新博客',
    url: 'caixin.com',
-    description: `> 网站部分内容需要付费订阅，RSS 仅做更新提醒，不含付费内容。`,
+    description: `> 网站部分内容需要付费订阅，RSS 仅做更新提醒，不含付费内容。若需要得到付费内容全文，请使用订阅账户在手机网页版登录，然后设置\`CAIXIN_COOKIE\`为至少包含cookie中的以下字段: \`SA_USER_UID\`, \`SA_USER_UNIT\`, \`SA_USER_DEVICE_TYPE\`, \`USER_LOGIN_CODE\``,
    lang: 'zh-CN',
 };
--- a/lib/routes/caixin/utils-fulltext.ts
+++ b/lib/routes/caixin/utils-fulltext.ts
@ -0,0 +1,49 @@
+import crypto from 'crypto';
+import { hextob64, KJUR } from 'jsrsasign';
+import ofetch from '@/utils/ofetch';
+import { config } from '@/config';
+
+
+// The following constant is extracted from this script: https://file.caixin.com/pkg/cx-pay-layer/js/wap.js?v=5.15.421933 . It is believed to contain no sensitive information.
+// Refer to this discussion for further explanation: https://github.com/DIYgod/RSSHub/pull/17231
+const rsaPrivateKey =
+    '-----BEGIN PRIVATE KEY-----MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCLci8q2u3NGFyFlMUwjCP91PsvGjHdRAq9fmqZLxvue+n+RhzNxnKKYOv35pLgFKWXsGq2TV+5Xrv6xZgNx36IUkqbmrO+eCa8NFmti04wvMfG3DCNdKA7Lue880daNiK3BOhlQlZPykUXt1NftMNS/z+e70W+Vpv1ZxCx5BipqZkdoceM3uin0vUQmqmHqjxi5qKUuov90dXLaMxypCA0TDsIDnX8RPvPtqKff1p2TMW2a0XYe7CPYhRggaQMpmo0TcFutgrM1Vywyr2TPxYR+H/tpuuWRET7tUIQykBYoO1WKfL2dX6cxarjAJfnYnod3sMzppHouyp8Pt7gHVG7AgMBAAECggEAEFshSy6IrADKgWSUyH/3jMNZfwnchW6Ar/9O847CAPQJ2yhQIpa/Qpnhs58Y5S2myqcHrUBgFPcWp3BbyGn43naAh8XahWHEcVjWl/N6BV9vM1UKYN0oGikDR3dljCBDbCIoPBBO3WcFOaXoIpaqPmbwCG1aSdwQyPUA0UzG08eDbuHK6L5jvbe3xv5kLpWTVddrocW+SakbZRAX1Ykp7IujOce235nM7GOfoq4b8jmK5CLg6VIZGQV20wnn9YxuFOndRSjneFberzfzBMhVLpPsQ16M2xDLpZaDTggZnq2L6nZygds8Hda++ga3WbD3TcgjJNYuENu1S88IowYhSQKBgQDFqRA+38mo6KsxVDCNWcuEk2hSq8NEUzRHJpS7/QjZmEIYpFzDXgSGwhZJ0WNsQtaxJeBbc7B/OOqh8TL1reLl5AdTimS1OLHWVf/MUsLVS7Y82hx/hpYWxZnRSq41oI3P8FO/53FiQMYo2wbwqF6uQjB1y8h58aqL3OYpTH/5xQKBgQC0mobALJ+bU4nCPzkVDZuD6RyNWPwS1aE3+925wDSN2rJ0iLIb4N5czWZmHb66VlAtfGbp2q+amsCV4r6UR19A/y8k9SFB0mdtxix6mjEfaGhVJm4B1mkvsn0OHMAanKkohUvCjROQc3sziyp2gqSEQ98G7//VMPx/3dhgyQpVfwKBgQCycsqu6N0n+D6t/0MCKiJaI7bYhCd7JN8aqVM4UN5PjG2Hz8PLwbK2cr0qkbaAA+vN7NMb3Vtn0FvMLnUCZqVlRTP0EQqQrYmoZuXUcpdhd8QkNgnqe/g+wND4qcKTucquA1uo8mtj9/Su5+bhGDC6hBk6D+uDZFHDiX/loyIavQKBgQCXF6AcLjjpDZ52b8Yloti0JtXIOuXILAlQeNoqiG5vLsOVUrcPM7VUFlLQo5no8kTpiOXgRyAaS9VKkAO4sW0zR0n9tUY5dvkokV6sw0rNZ9/BPQFTcDlXug99OvhMSzwJtlqHTNdNRg+QM6E2vF0+ejmf6DEz/mN/5e0cK5UFqQKBgCR2hVfbRtDz9Cm/P8chPqaWFkH5ulUxBpc704Igc6bVH5DrEoWo6akbeJixV2obAZO3sFyeJqBUqaCvqG17Xei6jn3Hc3WMz9nLrAJEI9BTCfwvuxCOyY0IxqAAYT28xYv42I4+ADT/PpCq2Dj5u43X0dapAjZBZDfVVis7q1Bw-----END PRIVATE KEY-----';
+
+export async function getFulltext(url: string) {
+    if (!config.caixin.cookie) {
+        return;
+    }
+
+    const nonce = crypto.randomUUID().replaceAll('-', '').toUpperCase();
+
+    const userID = config.caixin.cookie
+        .split(';')
+        .find((e) => e.includes('SA_USER_UID'))
+        ?.split('=')[1]; //
+
+    const articleID = url.match(/(\d+)\.html/)[1];
+    const rawString = `id=${articleID}&uid=${userID}&${nonce}=nonce`;
+
+    const sig = new KJUR.crypto.Signature({ alg: 'SHA256withRSA' });
+    sig.init(rsaPrivateKey);
+    sig.updateString(rawString);
+    const sigValueHex = hextob64(sig.sign());
+
+    const isWeekly = url.includes('weekly');
+    const res = await ofetch(`https://gateway.caixin.com/api/newauth/checkAuthByIdJsonp`, {
+        params: {
+            type: 1,
+            page: isWeekly ? 0 : 1,
+            rand: Math.random(),
+            id: articleID,
+        },
+        headers: {
+            'X-Sign': encodeURIComponent(sigValueHex),
+            'X-Nonce': encodeURIComponent(nonce),
+            Cookie: config.caixin.cookie,
+        },
+    });
+
+    const { content } = JSON.parse(res.data.match(/resetContentInfo\((.*)\)/)[1]);
+    return content;
+}
--- a/lib/routes/caixin/utils.ts
+++ b/lib/routes/caixin/utils.ts
@ -6,44 +6,44 @@ import { load } from 'cheerio';
 import { art } from '@/utils/render';
 import path from 'node:path';

-const parseArticle = (item, tryGet) =>
-    /\.blog\.caixin\.com$/.test(new URL(item.link).hostname)
-        ? parseBlogArticle(item, tryGet)
-        : tryGet(item.link, async () => {
-              const { data: response } = await got(item.link);
+const parseArticle = async (item) => {
+    if (/\.blog\.caixin\.com$/.test(new URL(item.link).hostname)) {
+        return parseBlogArticle(item);
+    } else {
+        const { data: response } = await got(item.link);

-              const $ = load(response);
+        const $ = load(response);

-              item.description = art(path.join(__dirname, 'templates/article.art'), {
-                  item,
-                  $,
-              });
+        item.description = art(path.join(__dirname, 'templates/article.art'), {
+            item,
+            $,
+        });

-              if (item.audio) {
-                  item.itunes_item_image = item.audio_image_url;
-                  item.enclosure_url = item.audio;
-                  item.enclosure_type = 'audio/mpeg';
-              }
-
-              return item;
-          });
-
-const parseBlogArticle = (item, tryGet) =>
-    tryGet(item.link, async () => {
-        const response = await got(item.link);
-        const $ = load(response.data);
-        const article = $('#the_content').removeAttr('style');
-        article.find('img').removeAttr('style');
-        article
-            .find('p')
-            // Non-breaking space U+00A0, `&nbsp;` in html
-            // element.children[0].data === $(element, article).text()
-            .filter((_, element) => element.children[0].data === String.fromCharCode(160))
-            .remove();
-
-        item.description = article.html();
+        if (item.audio) {
+            item.itunes_item_image = item.audio_image_url;
+            item.enclosure_url = item.audio;
+            item.enclosure_type = 'audio/mpeg';
+        }

        return item;
-    });
+    }
+};
+
+const parseBlogArticle = async (item) => {
+    const response = await got(item.link);
+    const $ = load(response.data);
+    const article = $('#the_content').removeAttr('style');
+    article.find('img').removeAttr('style');
+    article
+        .find('p')
+        // Non-breaking space U+00A0, `&nbsp;` in html
+        // element.children[0].data === $(element, article).text()
+        .filter((_, element) => element.children[0].data === String.fromCodePoint(160))
+        .remove();
+
+    item.description = article.html();
+
+    return item;
+};

 export { parseArticle, parseBlogArticle };