xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(cryptoslate): filter before parsing and fix contentEncoded parsing (#18509) 

* fix(cryptoslate): filter before parsing and fix contentEncoded parsing

* fix: description
This commit is contained in:
Tony 2025-03-04 17:07:20 +08:00 committed by GitHub
parent 40e4212e17
commit 2a25fdb6b7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 29 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
lib/routes/cryptoslate/index.ts
View File

@ -1,5 +1,4 @@
import { Route, Data } from '@/types';
import cache from '@/utils/cache';
import { parseDate } from '@/utils/parse-date';
import logger from '@/utils/logger';
import parser from '@/utils/rss-parser';
@ -18,7 +17,7 @@ export const route: Route = {
supportPodcast: false,
supportScihub: false,
},
name: 'CryptoSlate News',
name: 'News',
maintainers: ['pseudoyu'],
handler,
radar: [
@ -27,7 +26,7 @@ export const route: Route = {
target: '/',
},
],
description: `Get latest news from CryptoSlate.`,
description: 'Get latest news from CryptoSlate.',
};
async function handler(ctx): Promise<Data> {
@ -36,51 +35,33 @@ async function handler(ctx): Promise<Data> {
const feed = await parser.parseURL(rssUrl);
const items = await Promise.all(
feed.items.slice(0, limit).map((item) =>
cache.tryGet(`cryptoslate:article:${item.link || ''}`, async () => {
if (!item.link) {
return {};
}
const items = feed.items
.filter((item) => !item.link?.includes('/feed') && !item.link?.includes('#respond'))
.slice(0, limit)
.map((item) => {
if (!item.link) {
return {};
}
// Skip non-article content
if (item.link.includes('/feed') || item.link.includes('#respond')) {
return {};
}
try {
// Clean URL by removing query parameters
const cleanUrl = item.link.split('?')[0];
try {
// Clean URL by removing query parameters
const cleanUrl = item.link.split('?')[0];
const fullText = await Promise.resolve(extractFullTextFromRSS(item));
if (!fullText) {
return {};
}
// Get cover URL from media content
let coverUrl = null;
if ((item as any).media?.content && (item as any).media.content.length > 0) {
coverUrl = (item as any).media.content[0].url;
}
return {
title: item.title || 'Untitled',
link: cleanUrl,
pubDate: item.pubDate ? parseDate(item.pubDate) : undefined,
description: fullText,
author: item.creator || 'CryptoSlate',
category: item.categories || [],
guid: item.guid || item.link,
image: coverUrl,
};
} catch (error: any) {
logger.warn(`Couldn't process article from CryptoSlate: ${item.link}: ${error.message}`);
return {};
}
})
)
);
return {
title: item.title || 'Untitled',
link: cleanUrl,
pubDate: item.pubDate ? parseDate(item.pubDate) : undefined,
description: extractFullTextFromRSS(item),
author: item.creator || 'CryptoSlate',
category: item.categories || [],
guid: item.guid || item.link,
image: item.enclosure?.url,
};
} catch (error: any) {
logger.warn(`Couldn't process article from CryptoSlate: ${item.link}: ${error.message}`);
return {};
}
});
// Filter out empty items
const filteredItems = items.filter((item) => item && Object.keys(item).length > 0);
@ -97,8 +78,7 @@ async function handler(ctx): Promise<Data> {
function extractFullTextFromRSS(entry: any): string | null {
try {
// Get content from the RSS feed - match Python's approach precisely
const contentEncoded = entry.content && Array.isArray(entry.content) && entry.content.length > 0 ? entry.content[0].value : (typeof entry.content === 'string' ? entry.content : '');
const contentEncoded = entry['content:encoded'] || entry['content:encodedSnippet'] || entry.content || entry.contentSnippet;
if (!contentEncoded) {
return null;
@ -110,18 +90,7 @@ function extractFullTextFromRSS(entry: any): string | null {
$('img').remove();
$('figure').remove();
// Extract text from p and li elements
const allElements = $('p, li');
let fullText = '';
allElements.each((_, element) => {
const elementText = $(element).text().trim();
if (elementText) {
fullText += `<${element.name}>${$(element).html()}</${element.name}>`;
}
});
return fullText || null;
return $.html() || null;
} catch (error) {
logger.error(`Error extracting full text from RSS: ${error}`);
return null;