fix(route/openai): fetch OpenAI blog articles through RSS feed (#18952)
* fix(route/openai): fetch news through RSS feed * refactor: rename blog route to news * refactor(route/openai): ensure limit parameter works as expected * refactor(route/openai): use user agent from configuration * refactor(route/openai): cache data items * refactor(route/openai): cache individual data items * refactor: read query string parameter instead of path parameter
This commit is contained in:
@@ -1,66 +0,0 @@
|
||||
import { Route } from '@/types';
|
||||
import got from '@/utils/got';
|
||||
import { toTitleCase } from '@/utils/common-utils';
|
||||
import { getApiUrl, parseArticle } from './common';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/blog/:tag?',
|
||||
categories: ['programming'],
|
||||
example: '/openai/blog',
|
||||
parameters: { tag: 'Tag, see below, All by default' },
|
||||
features: {
|
||||
requireConfig: false,
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
name: 'Blog',
|
||||
maintainers: ['StevenRCE0', 'nczitzk'],
|
||||
handler,
|
||||
description: `| All | Announcements | Events | Safety & Alignment | Community | Product | Culture & Careers | Milestones | Research |
|
||||
| --- | ------------- | ------ | ------------------ | --------- | ------- | ------------------- | ---------- | -------- |
|
||||
| | announcements | events | safety-alignment | community | product | culture-and-careers | milestones | research |`,
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
const tag = ctx.req.param('tag') || '';
|
||||
|
||||
const rootUrl = 'https://openai.com';
|
||||
const blogRootUrl = 'https://openai.com/blog';
|
||||
const blogOriginUrl = `${rootUrl}/blog${tag === '' ? '' : `?topics=${tag}`}`;
|
||||
|
||||
const apiUrl = new URL('/api/v1/blog-details', await getApiUrl());
|
||||
|
||||
// Construct API query
|
||||
apiUrl.searchParams.append('sort', '-publicationDate,-createdAt');
|
||||
apiUrl.searchParams.append('page[size]', '20');
|
||||
apiUrl.searchParams.append('page[number]', '1');
|
||||
apiUrl.searchParams.append('include', 'media,topics,authors');
|
||||
if (tag) {
|
||||
apiUrl.searchParams.append('filter[topics][slugs][0]', tag);
|
||||
}
|
||||
|
||||
const response = await got({
|
||||
method: 'get',
|
||||
url: apiUrl,
|
||||
});
|
||||
|
||||
const list = response.data.data.filter((entry) => entry.type === 'blog-details');
|
||||
|
||||
const items = await Promise.all(
|
||||
list.map((item) => {
|
||||
const attributes = item.attributes;
|
||||
return parseArticle(ctx, blogRootUrl, attributes);
|
||||
})
|
||||
);
|
||||
|
||||
const title = `OpenAI Blog${tag ? ` - ${toTitleCase(tag)}` : ''}`;
|
||||
|
||||
return {
|
||||
title,
|
||||
link: blogOriginUrl,
|
||||
item: items,
|
||||
};
|
||||
}
|
||||
@@ -1,8 +1,77 @@
|
||||
import cache from '@/utils/cache';
|
||||
import got from '@/utils/got';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { DataItem } from '@/types';
|
||||
import { load } from 'cheerio';
|
||||
import { art } from '@/utils/render';
|
||||
import path from 'node:path';
|
||||
import { config } from '@/config';
|
||||
|
||||
export const BASE_URL = new URL('https://openai.com');
|
||||
|
||||
/** Fetch the details of an article. */
|
||||
export const fetchArticleDetails = async (url: string) => {
|
||||
const page = await ofetch(url);
|
||||
const $ = load(page);
|
||||
|
||||
const $article = $('#main article');
|
||||
|
||||
const categories = $('h1')
|
||||
.prev()
|
||||
.find('a[href]')
|
||||
.toArray()
|
||||
.map((element) => $(element).text());
|
||||
|
||||
// Article header (title, sub title and categories)
|
||||
$($article.find('h1').parents().get(4)).remove();
|
||||
// Related articles (can be the #citations section in some cases, so the last child needs to be removed first)
|
||||
$article.children().last().remove();
|
||||
// Article authors and tags
|
||||
$article.find('#citations').remove();
|
||||
|
||||
return {
|
||||
content: $article.html() ?? undefined,
|
||||
// Categories can be found on https://openai.com/news/ and https://openai.com/research/index/
|
||||
categories,
|
||||
image: $('meta[property="og:image"]').attr('content'),
|
||||
};
|
||||
};
|
||||
|
||||
/** Fetch all articles from OpenAI's RSS feed. */
|
||||
export const fetchArticles = async (limit: number): Promise<DataItem[]> => {
|
||||
const page = await ofetch('https://openai.com/news/rss.xml', {
|
||||
responseType: 'text',
|
||||
headers: { 'User-Agent': config.ua },
|
||||
});
|
||||
|
||||
const $ = load(page, { xml: true });
|
||||
|
||||
return Promise.all(
|
||||
$('item')
|
||||
.toArray()
|
||||
.slice(0, limit)
|
||||
.map<Promise<DataItem>>((element) => {
|
||||
const id = $(element).find('guid').text();
|
||||
|
||||
return cache.tryGet(`openai:news:${id}`, async () => {
|
||||
const title = $(element).find('title').text();
|
||||
const pubDate = $(element).find('pubDate').text();
|
||||
const link = $(element).find('link').text();
|
||||
|
||||
const { content, categories } = await fetchArticleDetails(link);
|
||||
|
||||
return {
|
||||
guid: id,
|
||||
title,
|
||||
link,
|
||||
pubDate,
|
||||
description: content,
|
||||
category: categories,
|
||||
} as DataItem;
|
||||
}) as Promise<DataItem>;
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
const getApiUrl = async () => {
|
||||
const blogRootUrl = 'https://openai.com/blog';
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { Route } from '@/types';
|
||||
import { fetchArticles, BASE_URL } from './common';
|
||||
import { Context } from 'hono';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/news',
|
||||
categories: ['programming'],
|
||||
example: '/openai/news',
|
||||
features: {
|
||||
requireConfig: false,
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
name: 'News',
|
||||
maintainers: ['goestav', 'StevenRCE0', 'nczitzk'],
|
||||
handler,
|
||||
};
|
||||
|
||||
async function handler(ctx: Context) {
|
||||
const limit = Number.parseInt(ctx.req.query('limit') || '10');
|
||||
|
||||
const link = new URL('/news/', BASE_URL).href;
|
||||
|
||||
return {
|
||||
title: 'OpenAI News',
|
||||
link,
|
||||
item: await fetchArticles(limit),
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user