fix(coindesk): consensus magazine (#18518)
* fix: coindesk consensus magazine * fix: unify article parsing * fix: use https
This commit is contained in:
Tony
GitHub
parent
54bd58b002
commit
1cce4bf785
|
|
@ -1,6 +1,8 @@
|
|||
import { Route } from '@/types';
|
||||
import got from '@/utils/got';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import cache from '@/utils/cache';
|
||||
import { load } from 'cheerio';
|
||||
import { parseItem } from './utils';
|
||||
const rootUrl = 'https://www.coindesk.com';
|
||||
|
||||
export const route: Route = {
|
||||
|
|
@ -27,29 +29,23 @@ export const route: Route = {
|
|||
url: 'coindesk.com/',
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
const channel = ctx.req.param('channel') ?? 'consensus-magazine';
|
||||
async function handler() {
|
||||
const channel = 'consensus-magazine';
|
||||
|
||||
const response = await got.get(`${rootUrl}/${channel}/`);
|
||||
const $ = load(response.data);
|
||||
const content = JSON.parse(
|
||||
$('#fusion-metadata')
|
||||
.text()
|
||||
.match(/Fusion\.contentCache=(.*?);Fusion\.layout/)[1]
|
||||
);
|
||||
const response = await ofetch(`${rootUrl}/${channel}`);
|
||||
const $ = load(response);
|
||||
|
||||
const o1 = content['websked-collections'];
|
||||
// Object key names are different every week
|
||||
const articles = o1[Object.keys(o1)[2]];
|
||||
const list = $('div h2')
|
||||
.toArray()
|
||||
.map((item) => {
|
||||
const $item = $(item);
|
||||
return {
|
||||
title: $item.text(),
|
||||
link: rootUrl + $item.parent().attr('href'),
|
||||
};
|
||||
});
|
||||
|
||||
const list = articles.data;
|
||||
|
||||
const items = list.map((item) => ({
|
||||
title: item.headlines.basic,
|
||||
link: rootUrl + item.canonical_url,
|
||||
description: item.subheadlines.basic,
|
||||
pubDate: item.display_date,
|
||||
}));
|
||||
const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => parseItem(item))));
|
||||
|
||||
return {
|
||||
title: 'CoinDesk Consensus Magazine',
|
||||
|
|
@ -1,10 +1,7 @@
|
|||
import { Route, Data, DataItem } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { load } from 'cheerio';
|
||||
import logger from '@/utils/logger';
|
||||
import parser from '@/utils/rss-parser';
|
||||
import { parseItem } from './utils';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/news',
|
||||
|
|
@ -19,7 +16,7 @@ export const route: Route = {
|
|||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
name: 'CoinDesk News',
|
||||
name: 'News',
|
||||
maintainers: ['pseudoyu'],
|
||||
handler,
|
||||
radar: [
|
||||
|
|
@ -28,60 +25,17 @@ export const route: Route = {
|
|||
target: '/news',
|
||||
},
|
||||
],
|
||||
description: `Get latest news from CoinDesk with full text.`,
|
||||
description: 'Get latest news from CoinDesk with full text.',
|
||||
};
|
||||
|
||||
async function handler(): Promise<Data> {
|
||||
const rssUrl = 'http://feeds.feedburner.com/Coindesk';
|
||||
const rssUrl = 'https://feeds.feedburner.com/Coindesk';
|
||||
const feed = await parser.parseURL(rssUrl);
|
||||
|
||||
const items = await Promise.all(
|
||||
feed.items.map(async (item) => {
|
||||
const link = item.link;
|
||||
if (!link) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cleanLink = link.split('?')[0];
|
||||
|
||||
// Get cover URL from media content
|
||||
let coverUrl: string | undefined;
|
||||
const mediaContent = (item as any).media?.content;
|
||||
if (mediaContent && mediaContent.length > 0) {
|
||||
const url = mediaContent[0].url;
|
||||
if (url) {
|
||||
// Extract the required part of the cover URL
|
||||
const match = url.match(/https?:\/\/(?:www\.)?(?:\S+?\/)?([a-z]+-?\d+\.images\..+?\/coindesk\/.+)/i);
|
||||
coverUrl = match ? `https://${match[1]}` : url;
|
||||
}
|
||||
}
|
||||
|
||||
// Extract full text
|
||||
const fullText = await cache.tryGet(cleanLink, async () => {
|
||||
const text = await extractFullText(cleanLink);
|
||||
return text || '';
|
||||
});
|
||||
|
||||
if (!fullText) {
|
||||
logger.warn(`Failed to extract content from ${cleanLink}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Create article item
|
||||
return {
|
||||
title: item.title || 'Untitled',
|
||||
description: fullText,
|
||||
pubDate: item.pubDate ? parseDate(item.pubDate) : new Date(),
|
||||
link: cleanLink,
|
||||
author: item.creator || 'CoinDesk',
|
||||
category: item.categories || [],
|
||||
image: coverUrl,
|
||||
} as DataItem;
|
||||
})
|
||||
);
|
||||
const items = await Promise.all(feed.items.map((item) => cache.tryGet(item.link, () => parseItem(item))));
|
||||
|
||||
// Filter out null items
|
||||
const validItems = items.filter((item): item is NonNullable<typeof item> => item !== null);
|
||||
const validItems = items.filter((item): item is DataItem => item !== null);
|
||||
|
||||
return {
|
||||
title: feed.title || 'CoinDesk News',
|
||||
|
|
@ -91,35 +45,3 @@ async function handler(): Promise<Data> {
|
|||
item: validItems,
|
||||
};
|
||||
}
|
||||
|
||||
async function extractFullText(url: string): Promise<string | null> {
|
||||
try {
|
||||
const response = await ofetch(url);
|
||||
const $ = load(response);
|
||||
const article = $('div[data-module-name="article-body"]');
|
||||
|
||||
if (!article.length) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Remove unwanted elements
|
||||
article.find('div.article__badge').remove();
|
||||
article.find('div.article__share').remove();
|
||||
|
||||
// Extract text from paragraphs and list items
|
||||
const textElements = article.find('p, li');
|
||||
let fullText = '';
|
||||
|
||||
textElements.each((_, element) => {
|
||||
const text = $(element).text().trim();
|
||||
if (text) {
|
||||
fullText += `<p>${text}</p>`;
|
||||
}
|
||||
});
|
||||
|
||||
return fullText || null;
|
||||
} catch (error) {
|
||||
logger.error(`Error fetching article content: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
import ofetch from '@/utils/ofetch';
|
||||
import { load } from 'cheerio';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
export const parseItem = async (item) => {
|
||||
const response = await ofetch(item.link);
|
||||
const $ = load(response);
|
||||
const ldJson = JSON.parse($('script[type="application/ld+json"]').text());
|
||||
|
||||
$('.article-ad, #strategy-rules-player-wrapper, [data-module-name="newsletter-article-sign-up-module"], div.flex.flex-col.gap-2').remove();
|
||||
const cover = $('.article-content-wrapper figure');
|
||||
cover.find('img').attr('src', cover.find('img').attr('url')?.split('?')[0]);
|
||||
cover.find('img').removeAttr('style srcset url');
|
||||
|
||||
item.description =
|
||||
cover.parent().html() +
|
||||
$('.document-body')
|
||||
.toArray()
|
||||
.map((item) => $(item).html())
|
||||
.join('');
|
||||
item.pubDate = parseDate(ldJson.datePublished);
|
||||
item.author = ldJson.author.map((a) => ({ name: a.name }));
|
||||
item.image = ldJson.image.url.split('?')[0];
|
||||
|
||||
return item;
|
||||
};
|
||||
Loading…
Reference in New Issue