From 2a25fdb6b7ef0c621616705ceeb5546fe4ce048a Mon Sep 17 00:00:00 2001
From: Tony <TonyRL@users.noreply.github.com>
Date: Tue, 4 Mar 2025 17:07:20 +0800
Subject: [PATCH] fix(cryptoslate): filter before parsing and fix
 contentEncoded parsing (#18509)

* fix(cryptoslate): filter before parsing and fix contentEncoded parsing

* fix: description
---
 lib/routes/cryptoslate/index.ts | 89 +++++++++++----------------------
 1 file changed, 29 insertions(+), 60 deletions(-)

diff --git a/lib/routes/cryptoslate/index.ts b/lib/routes/cryptoslate/index.ts
index c8f7bb897..978411c33 100644
--- a/lib/routes/cryptoslate/index.ts
+++ b/lib/routes/cryptoslate/index.ts
@@ -1,5 +1,4 @@
 import { Route, Data } from '@/types';
-import cache from '@/utils/cache';
 import { parseDate } from '@/utils/parse-date';
 import logger from '@/utils/logger';
 import parser from '@/utils/rss-parser';
@@ -18,7 +17,7 @@ export const route: Route = {
         supportPodcast: false,
         supportScihub: false,
     },
-    name: 'CryptoSlate News',
+    name: 'News',
     maintainers: ['pseudoyu'],
     handler,
     radar: [
@@ -27,7 +26,7 @@ export const route: Route = {
             target: '/',
         },
     ],
-    description: `Get latest news from CryptoSlate.`,
+    description: 'Get latest news from CryptoSlate.',
 };
 
 async function handler(ctx): Promise<Data> {
@@ -36,51 +35,33 @@ async function handler(ctx): Promise<Data> {
 
     const feed = await parser.parseURL(rssUrl);
 
-    const items = await Promise.all(
-        feed.items.slice(0, limit).map((item) =>
-            cache.tryGet(`cryptoslate:article:${item.link || ''}`, async () => {
-                if (!item.link) {
-                    return {};
-                }
+    const items = feed.items
+        .filter((item) => !item.link?.includes('/feed') && !item.link?.includes('#respond'))
+        .slice(0, limit)
+        .map((item) => {
+            if (!item.link) {
+                return {};
+            }
 
-                // Skip non-article content
-                if (item.link.includes('/feed') || item.link.includes('#respond')) {
-                    return {};
-                }
+            try {
+                // Clean URL by removing query parameters
+                const cleanUrl = item.link.split('?')[0];
 
-                try {
-                    // Clean URL by removing query parameters
-                    const cleanUrl = item.link.split('?')[0];
-
-                    const fullText = await Promise.resolve(extractFullTextFromRSS(item));
-
-                    if (!fullText) {
-                        return {};
-                    }
-
-                    // Get cover URL from media content
-                    let coverUrl = null;
-                    if ((item as any).media?.content && (item as any).media.content.length > 0) {
-                        coverUrl = (item as any).media.content[0].url;
-                    }
-
-                    return {
-                        title: item.title || 'Untitled',
-                        link: cleanUrl,
-                        pubDate: item.pubDate ? parseDate(item.pubDate) : undefined,
-                        description: fullText,
-                        author: item.creator || 'CryptoSlate',
-                        category: item.categories || [],
-                        guid: item.guid || item.link,
-                        image: coverUrl,
-                    };
-                } catch (error: any) {
-                    logger.warn(`Couldn't process article from CryptoSlate: ${item.link}: ${error.message}`);
-                    return {};
-                }
-            })
-        )
-    );
+                return {
+                    title: item.title || 'Untitled',
+                    link: cleanUrl,
+                    pubDate: item.pubDate ? parseDate(item.pubDate) : undefined,
+                    description: extractFullTextFromRSS(item),
+                    author: item.creator || 'CryptoSlate',
+                    category: item.categories || [],
+                    guid: item.guid || item.link,
+                    image: item.enclosure?.url,
+                };
+            } catch (error: any) {
+                logger.warn(`Couldn't process article from CryptoSlate: ${item.link}: ${error.message}`);
+                return {};
+            }
+        });
 
     // Filter out empty items
     const filteredItems = items.filter((item) => item && Object.keys(item).length > 0);
@@ -97,8 +78,7 @@ async function handler(ctx): Promise<Data> {
 
 function extractFullTextFromRSS(entry: any): string | null {
     try {
-        // Get content from the RSS feed - match Python's approach precisely
-        const contentEncoded = entry.content && Array.isArray(entry.content) && entry.content.length > 0 ? entry.content[0].value : (typeof entry.content === 'string' ? entry.content : '');
+        const contentEncoded = entry['content:encoded'] || entry['content:encodedSnippet'] || entry.content || entry.contentSnippet;
 
         if (!contentEncoded) {
             return null;
@@ -110,18 +90,7 @@ function extractFullTextFromRSS(entry: any): string | null {
         $('img').remove();
         $('figure').remove();
 
-        // Extract text from p and li elements
-        const allElements = $('p, li');
-        let fullText = '';
-
-        allElements.each((_, element) => {
-            const elementText = $(element).text().trim();
-            if (elementText) {
-                fullText += `<${element.name}>${$(element).html()}</${element.name}>`;
-            }
-        });
-
-        return fullText || null;
+        return $.html() || null;
     } catch (error) {
         logger.error(`Error extracting full text from RSS: ${error}`);
         return null;