fix(blockworks): use next data api (#18521)
* fix(blockworks): use next data api * fix: limit params
This commit is contained in:
Tony
GitHub
parent
011af84d1d
commit
92cc3e6f49
|
|
@ -5,6 +5,7 @@ import { parseDate } from '@/utils/parse-date';
|
|||
import { load } from 'cheerio';
|
||||
import logger from '@/utils/logger';
|
||||
import parser from '@/utils/rss-parser';
|
||||
import { config } from '@/config';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/',
|
||||
|
|
@ -25,44 +26,47 @@ export const route: Route = {
|
|||
target: '/',
|
||||
},
|
||||
],
|
||||
name: 'Blockworks News',
|
||||
name: 'News',
|
||||
maintainers: ['pseudoyu'],
|
||||
handler,
|
||||
description: `Blockworks news with full text support.`,
|
||||
description: 'Blockworks news with full text support.',
|
||||
};
|
||||
|
||||
async function handler(): Promise<Data> {
|
||||
async function handler(ctx): Promise<Data> {
|
||||
const rssUrl = 'https://blockworks.co/feed';
|
||||
const feed = await parser.parseURL(rssUrl);
|
||||
|
||||
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20;
|
||||
// Limit to 20 items
|
||||
const limitedItems = feed.items.slice(0, 20);
|
||||
const limitedItems = feed.items.slice(0, limit);
|
||||
|
||||
const buildId = await getBuildId();
|
||||
|
||||
const items = await Promise.all(
|
||||
limitedItems.map(async (item) => {
|
||||
const articleUrl = item.link || '';
|
||||
const cleanUrl = articleUrl.split('?')[0];
|
||||
limitedItems
|
||||
.map((item) => ({
|
||||
...item,
|
||||
link: item.link?.split('?')[0],
|
||||
}))
|
||||
.map((item) =>
|
||||
cache.tryGet(item.link!, async () => {
|
||||
// Get cached content or fetch new content
|
||||
const content = await extractFullText(item.link!.split('/').pop()!, buildId);
|
||||
|
||||
// Get cached content or fetch new content
|
||||
const fullText = await cache.tryGet(`blockworks:${cleanUrl}`, () => extractFullText(cleanUrl));
|
||||
|
||||
// Get cover image
|
||||
const coverUrl = await cache.tryGet(`blockworks:cover:${cleanUrl}`, () => extractCoverUrl(cleanUrl));
|
||||
|
||||
return {
|
||||
title: item.title || 'Untitled',
|
||||
pubDate: item.isoDate ? parseDate(item.isoDate) : new Date(),
|
||||
link: cleanUrl,
|
||||
description: fullText || item.content || item.summary || '',
|
||||
author: item.creator,
|
||||
category: item.categories,
|
||||
media: coverUrl
|
||||
? {
|
||||
content: { url: coverUrl },
|
||||
}
|
||||
: undefined,
|
||||
} as DataItem;
|
||||
})
|
||||
return {
|
||||
title: item.title || 'Untitled',
|
||||
pubDate: item.isoDate ? parseDate(item.isoDate) : undefined,
|
||||
link: item.link,
|
||||
description: content.description || item.content || item.contentSnippet || item.summary || '',
|
||||
author: item.author,
|
||||
category: content.category,
|
||||
media: content.imageUrl
|
||||
? {
|
||||
content: { url: content.imageUrl },
|
||||
}
|
||||
: undefined,
|
||||
} as DataItem;
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
return {
|
||||
|
|
@ -74,67 +78,51 @@ async function handler(): Promise<Data> {
|
|||
};
|
||||
}
|
||||
|
||||
async function extractFullText(url: string): Promise<string> {
|
||||
async function extractFullText(slug: string, buildId: string): Promise<{ description: string; imageUrl: string; category: string[] }> {
|
||||
try {
|
||||
const response = await ofetch(url);
|
||||
const $ = load(response);
|
||||
const response = await ofetch(`https://blockworks.co/_next/data/${buildId}/news/${slug}.json?slug=${slug}`);
|
||||
const article = response.pageProps.article;
|
||||
const $ = load(article.content, null, false);
|
||||
|
||||
const articleSection = $('section.w-full');
|
||||
|
||||
if (articleSection.length) {
|
||||
// Extract all paragraph and list items
|
||||
const allElements = articleSection.find('p, li');
|
||||
let text = '';
|
||||
|
||||
allElements.each((_, element) => {
|
||||
const content = $(element).text().trim();
|
||||
if (content) {
|
||||
text += content + '\n\n';
|
||||
}
|
||||
});
|
||||
|
||||
// Remove promotional content at the end
|
||||
const promoStartKeywords = ["Get the day's top crypto", 'Want alpha sent', "Can't wait?"];
|
||||
|
||||
for (const keyword of promoStartKeywords) {
|
||||
const index = text.indexOf(keyword);
|
||||
if (index !== -1) {
|
||||
text = text.substring(0, index).trim();
|
||||
}
|
||||
// Remove promotional content at the end
|
||||
$('hr').remove();
|
||||
$('p > em, p > strong').each((_, el) => {
|
||||
const $el = $(el);
|
||||
if ($el.text().includes('To read full editions') || $el.text().includes('Get the news in your inbox')) {
|
||||
$el.parent().remove();
|
||||
}
|
||||
});
|
||||
$('ul.wp-block-list > li > a').each((_, el) => {
|
||||
const $el = $(el);
|
||||
if ($el.attr('href') === 'https://blockworks.co/newsletter/daily') {
|
||||
$el.parent().parent().remove();
|
||||
}
|
||||
});
|
||||
|
||||
return text || '';
|
||||
}
|
||||
|
||||
return '';
|
||||
return {
|
||||
description: $.html(),
|
||||
imageUrl: article.imageUrl,
|
||||
category: [...new Set([...article.categories, ...article.tags])],
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Error extracting full text from Blockworks:', error);
|
||||
return '';
|
||||
return { description: '', imageUrl: '', category: [] };
|
||||
}
|
||||
}
|
||||
|
||||
async function extractCoverUrl(url: string): Promise<string> {
|
||||
try {
|
||||
const response = await ofetch(url);
|
||||
const $ = load(response);
|
||||
const getBuildId = () =>
|
||||
cache.tryGet(
|
||||
'blockworks:buildId',
|
||||
async () => {
|
||||
const response = await ofetch('https://blockworks.co');
|
||||
const $ = load(response);
|
||||
|
||||
const imageContainer = $('div.w-full.relative');
|
||||
if (imageContainer.length) {
|
||||
const imgTag = imageContainer.find('img[alt="article-image"]');
|
||||
if (imgTag.length && imgTag.attr('src')) {
|
||||
let imgUrl = imgTag.attr('src') as string;
|
||||
|
||||
if (!imgUrl.startsWith('http')) {
|
||||
imgUrl = `https://blockworks.co${imgUrl}`;
|
||||
}
|
||||
|
||||
return decodeURIComponent(imgUrl);
|
||||
}
|
||||
}
|
||||
|
||||
return '';
|
||||
} catch (error) {
|
||||
logger.error('Error extracting cover URL from Blockworks:', error);
|
||||
return '';
|
||||
}
|
||||
}
|
||||
return (
|
||||
$('script#__NEXT_DATA__')
|
||||
.text()
|
||||
?.match(/"buildId":"(.*?)",/)?.[1] || ''
|
||||
);
|
||||
},
|
||||
config.cache.routeExpire,
|
||||
false
|
||||
) as Promise<string>;
|
||||
|
|
|
|||
Loading…
Reference in New Issue