From 1cce4bf7853d3e39a2a7d9d6d5e899f08e856872 Mon Sep 17 00:00:00 2001
From: Tony <TonyRL@users.noreply.github.com>
Date: Tue, 4 Mar 2025 19:36:34 +0800
Subject: [PATCH] fix(coindesk): consensus magazine (#18518)

* fix: coindesk consensus magazine

* fix: unify article parsing

* fix: use https
---
 .../{index.ts => consensus-magazine.ts}       | 38 ++++----
 lib/routes/coindesk/news.ts                   | 90 ++-----------------
 lib/routes/coindesk/utils.ts                  | 26 ++++++
 3 files changed, 49 insertions(+), 105 deletions(-)
 rename lib/routes/coindesk/{index.ts => consensus-magazine.ts} (50%)
 create mode 100644 lib/routes/coindesk/utils.ts

diff --git a/lib/routes/coindesk/index.ts b/lib/routes/coindesk/consensus-magazine.ts
similarity index 50%
rename from lib/routes/coindesk/index.ts
rename to lib/routes/coindesk/consensus-magazine.ts
index 014e99ca5..3ae06e973 100644
--- a/lib/routes/coindesk/index.ts
+++ b/lib/routes/coindesk/consensus-magazine.ts
@@ -1,6 +1,8 @@
 import { Route } from '@/types';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
+import cache from '@/utils/cache';
 import { load } from 'cheerio';
+import { parseItem } from './utils';
 const rootUrl = 'https://www.coindesk.com';
 
 export const route: Route = {
@@ -27,29 +29,23 @@ export const route: Route = {
     url: 'coindesk.com/',
 };
 
-async function handler(ctx) {
-    const channel = ctx.req.param('channel') ?? 'consensus-magazine';
+async function handler() {
+    const channel = 'consensus-magazine';
 
-    const response = await got.get(`${rootUrl}/${channel}/`);
-    const $ = load(response.data);
-    const content = JSON.parse(
-        $('#fusion-metadata')
-            .text()
-            .match(/Fusion\.contentCache=(.*?);Fusion\.layout/)[1]
-    );
+    const response = await ofetch(`${rootUrl}/${channel}`);
+    const $ = load(response);
 
-    const o1 = content['websked-collections'];
-    // Object key names are different every week
-    const articles = o1[Object.keys(o1)[2]];
+    const list = $('div h2')
+        .toArray()
+        .map((item) => {
+            const $item = $(item);
+            return {
+                title: $item.text(),
+                link: rootUrl + $item.parent().attr('href'),
+            };
+        });
 
-    const list = articles.data;
-
-    const items = list.map((item) => ({
-        title: item.headlines.basic,
-        link: rootUrl + item.canonical_url,
-        description: item.subheadlines.basic,
-        pubDate: item.display_date,
-    }));
+    const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => parseItem(item))));
 
     return {
         title: 'CoinDesk Consensus Magazine',
diff --git a/lib/routes/coindesk/news.ts b/lib/routes/coindesk/news.ts
index 036d8d03b..42adf1da8 100644
--- a/lib/routes/coindesk/news.ts
+++ b/lib/routes/coindesk/news.ts
@@ -1,10 +1,7 @@
 import { Route, Data, DataItem } from '@/types';
 import cache from '@/utils/cache';
-import ofetch from '@/utils/ofetch';
-import { parseDate } from '@/utils/parse-date';
-import { load } from 'cheerio';
-import logger from '@/utils/logger';
 import parser from '@/utils/rss-parser';
+import { parseItem } from './utils';
 
 export const route: Route = {
     path: '/news',
@@ -19,7 +16,7 @@ export const route: Route = {
         supportPodcast: false,
         supportScihub: false,
     },
-    name: 'CoinDesk News',
+    name: 'News',
     maintainers: ['pseudoyu'],
     handler,
     radar: [
@@ -28,60 +25,17 @@ export const route: Route = {
             target: '/news',
         },
     ],
-    description: `Get latest news from CoinDesk with full text.`,
+    description: 'Get latest news from CoinDesk with full text.',
 };
 
 async function handler(): Promise<Data> {
-    const rssUrl = 'http://feeds.feedburner.com/Coindesk';
+    const rssUrl = 'https://feeds.feedburner.com/Coindesk';
     const feed = await parser.parseURL(rssUrl);
 
-    const items = await Promise.all(
-        feed.items.map(async (item) => {
-            const link = item.link;
-            if (!link) {
-                return null;
-            }
-
-            const cleanLink = link.split('?')[0];
-
-            // Get cover URL from media content
-            let coverUrl: string | undefined;
-            const mediaContent = (item as any).media?.content;
-            if (mediaContent && mediaContent.length > 0) {
-                const url = mediaContent[0].url;
-                if (url) {
-                    // Extract the required part of the cover URL
-                    const match = url.match(/https?:\/\/(?:www\.)?(?:\S+?\/)?([a-z]+-?\d+\.images\..+?\/coindesk\/.+)/i);
-                    coverUrl = match ? `https://${match[1]}` : url;
-                }
-            }
-
-            // Extract full text
-            const fullText = await cache.tryGet(cleanLink, async () => {
-                const text = await extractFullText(cleanLink);
-                return text || '';
-            });
-
-            if (!fullText) {
-                logger.warn(`Failed to extract content from ${cleanLink}`);
-                return null;
-            }
-
-            // Create article item
-            return {
-                title: item.title || 'Untitled',
-                description: fullText,
-                pubDate: item.pubDate ? parseDate(item.pubDate) : new Date(),
-                link: cleanLink,
-                author: item.creator || 'CoinDesk',
-                category: item.categories || [],
-                image: coverUrl,
-            } as DataItem;
-        })
-    );
+    const items = await Promise.all(feed.items.map((item) => cache.tryGet(item.link, () => parseItem(item))));
 
     // Filter out null items
-    const validItems = items.filter((item): item is NonNullable<typeof item> => item !== null);
+    const validItems = items.filter((item): item is DataItem => item !== null);
 
     return {
         title: feed.title || 'CoinDesk News',
@@ -91,35 +45,3 @@ async function handler(): Promise<Data> {
         item: validItems,
     };
 }
-
-async function extractFullText(url: string): Promise<string | null> {
-    try {
-        const response = await ofetch(url);
-        const $ = load(response);
-        const article = $('div[data-module-name="article-body"]');
-
-        if (!article.length) {
-            return null;
-        }
-
-        // Remove unwanted elements
-        article.find('div.article__badge').remove();
-        article.find('div.article__share').remove();
-
-        // Extract text from paragraphs and list items
-        const textElements = article.find('p, li');
-        let fullText = '';
-
-        textElements.each((_, element) => {
-            const text = $(element).text().trim();
-            if (text) {
-                fullText += `<p>${text}</p>`;
-            }
-        });
-
-        return fullText || null;
-    } catch (error) {
-        logger.error(`Error fetching article content: ${error}`);
-        return null;
-    }
-}
diff --git a/lib/routes/coindesk/utils.ts b/lib/routes/coindesk/utils.ts
new file mode 100644
index 000000000..d43d91f8e
--- /dev/null
+++ b/lib/routes/coindesk/utils.ts
@@ -0,0 +1,26 @@
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+export const parseItem = async (item) => {
+    const response = await ofetch(item.link);
+    const $ = load(response);
+    const ldJson = JSON.parse($('script[type="application/ld+json"]').text());
+
+    $('.article-ad, #strategy-rules-player-wrapper, [data-module-name="newsletter-article-sign-up-module"], div.flex.flex-col.gap-2').remove();
+    const cover = $('.article-content-wrapper figure');
+    cover.find('img').attr('src', cover.find('img').attr('url')?.split('?')[0]);
+    cover.find('img').removeAttr('style srcset url');
+
+    item.description =
+        cover.parent().html() +
+        $('.document-body')
+            .toArray()
+            .map((item) => $(item).html())
+            .join('');
+    item.pubDate = parseDate(ldJson.datePublished);
+    item.author = ldJson.author.map((a) => ({ name: a.name }));
+    item.image = ldJson.image.url.split('?')[0];
+
+    return item;
+};