fix(route/openai): fetch OpenAI blog articles through RSS feed (#18952)

* fix(route/openai): fetch news through RSS feed

* refactor: rename blog route to news

* refactor(route/openai): ensure limit parameter works as expected

* refactor(route/openai): use user agent from configuration

* refactor(route/openai): cache data items

* refactor(route/openai): cache individual data items

* refactor: read query string parameter instead of path parameter
This commit is contained in:
Goestav
2025-04-28 03:23:19 +08:00
committed by GitHub
parent 10288db67d
commit e543a92ee6
3 changed files with 101 additions and 66 deletions
-66
View File
@@ -1,66 +0,0 @@
import { Route } from '@/types';
import got from '@/utils/got';
import { toTitleCase } from '@/utils/common-utils';
import { getApiUrl, parseArticle } from './common';
export const route: Route = {
path: '/blog/:tag?',
categories: ['programming'],
example: '/openai/blog',
parameters: { tag: 'Tag, see below, All by default' },
features: {
requireConfig: false,
requirePuppeteer: false,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
supportScihub: false,
},
name: 'Blog',
maintainers: ['StevenRCE0', 'nczitzk'],
handler,
description: `| All | Announcements | Events | Safety & Alignment | Community | Product | Culture & Careers | Milestones | Research |
| --- | ------------- | ------ | ------------------ | --------- | ------- | ------------------- | ---------- | -------- |
| | announcements | events | safety-alignment | community | product | culture-and-careers | milestones | research |`,
};
async function handler(ctx) {
const tag = ctx.req.param('tag') || '';
const rootUrl = 'https://openai.com';
const blogRootUrl = 'https://openai.com/blog';
const blogOriginUrl = `${rootUrl}/blog${tag === '' ? '' : `?topics=${tag}`}`;
const apiUrl = new URL('/api/v1/blog-details', await getApiUrl());
// Construct API query
apiUrl.searchParams.append('sort', '-publicationDate,-createdAt');
apiUrl.searchParams.append('page[size]', '20');
apiUrl.searchParams.append('page[number]', '1');
apiUrl.searchParams.append('include', 'media,topics,authors');
if (tag) {
apiUrl.searchParams.append('filter[topics][slugs][0]', tag);
}
const response = await got({
method: 'get',
url: apiUrl,
});
const list = response.data.data.filter((entry) => entry.type === 'blog-details');
const items = await Promise.all(
list.map((item) => {
const attributes = item.attributes;
return parseArticle(ctx, blogRootUrl, attributes);
})
);
const title = `OpenAI Blog${tag ? ` - ${toTitleCase(tag)}` : ''}`;
return {
title,
link: blogOriginUrl,
item: items,
};
}
+69
View File
@@ -1,8 +1,77 @@
import cache from '@/utils/cache';
import got from '@/utils/got';
import ofetch from '@/utils/ofetch';
import { DataItem } from '@/types';
import { load } from 'cheerio';
import { art } from '@/utils/render';
import path from 'node:path';
import { config } from '@/config';
export const BASE_URL = new URL('https://openai.com');
/** Fetch the details of an article. */
export const fetchArticleDetails = async (url: string) => {
const page = await ofetch(url);
const $ = load(page);
const $article = $('#main article');
const categories = $('h1')
.prev()
.find('a[href]')
.toArray()
.map((element) => $(element).text());
// Article header (title, sub title and categories)
$($article.find('h1').parents().get(4)).remove();
// Related articles (can be the #citations section in some cases, so the last child needs to be removed first)
$article.children().last().remove();
// Article authors and tags
$article.find('#citations').remove();
return {
content: $article.html() ?? undefined,
// Categories can be found on https://openai.com/news/ and https://openai.com/research/index/
categories,
image: $('meta[property="og:image"]').attr('content'),
};
};
/** Fetch all articles from OpenAI's RSS feed. */
export const fetchArticles = async (limit: number): Promise<DataItem[]> => {
const page = await ofetch('https://openai.com/news/rss.xml', {
responseType: 'text',
headers: { 'User-Agent': config.ua },
});
const $ = load(page, { xml: true });
return Promise.all(
$('item')
.toArray()
.slice(0, limit)
.map<Promise<DataItem>>((element) => {
const id = $(element).find('guid').text();
return cache.tryGet(`openai:news:${id}`, async () => {
const title = $(element).find('title').text();
const pubDate = $(element).find('pubDate').text();
const link = $(element).find('link').text();
const { content, categories } = await fetchArticleDetails(link);
return {
guid: id,
title,
link,
pubDate,
description: content,
category: categories,
} as DataItem;
}) as Promise<DataItem>;
})
);
};
const getApiUrl = async () => {
const blogRootUrl = 'https://openai.com/blog';
+32
View File
@@ -0,0 +1,32 @@
import { Route } from '@/types';
import { fetchArticles, BASE_URL } from './common';
import { Context } from 'hono';
export const route: Route = {
path: '/news',
categories: ['programming'],
example: '/openai/news',
features: {
requireConfig: false,
requirePuppeteer: false,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
supportScihub: false,
},
name: 'News',
maintainers: ['goestav', 'StevenRCE0', 'nczitzk'],
handler,
};
async function handler(ctx: Context) {
const limit = Number.parseInt(ctx.req.query('limit') || '10');
const link = new URL('/news/', BASE_URL).href;
return {
title: 'OpenAI News',
link,
item: await fetchArticles(limit),
};
}