From e543a92ee68622d0a97960af49d2d33ddebfde4b Mon Sep 17 00:00:00 2001 From: Goestav <27970303+goestav@users.noreply.github.com> Date: Sun, 27 Apr 2025 19:23:19 +0000 Subject: [PATCH] fix(route/openai): fetch OpenAI blog articles through RSS feed (#18952) * fix(route/openai): fetch news through RSS feed * refactor: rename blog route to news * refactor(route/openai): ensure limit parameter works as expected * refactor(route/openai): use user agent from configuration * refactor(route/openai): cache data items * refactor(route/openai): cache individual data items * refactor: read query string parameter instead of path parameter --- lib/routes/openai/blog.ts | 66 ----------------------------------- lib/routes/openai/common.ts | 69 +++++++++++++++++++++++++++++++++++++ lib/routes/openai/news.ts | 32 +++++++++++++++++ 3 files changed, 101 insertions(+), 66 deletions(-) delete mode 100644 lib/routes/openai/blog.ts create mode 100644 lib/routes/openai/news.ts diff --git a/lib/routes/openai/blog.ts b/lib/routes/openai/blog.ts deleted file mode 100644 index 1b1ea68a6..000000000 --- a/lib/routes/openai/blog.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { Route } from '@/types'; -import got from '@/utils/got'; -import { toTitleCase } from '@/utils/common-utils'; -import { getApiUrl, parseArticle } from './common'; - -export const route: Route = { - path: '/blog/:tag?', - categories: ['programming'], - example: '/openai/blog', - parameters: { tag: 'Tag, see below, All by default' }, - features: { - requireConfig: false, - requirePuppeteer: false, - antiCrawler: false, - supportBT: false, - supportPodcast: false, - supportScihub: false, - }, - name: 'Blog', - maintainers: ['StevenRCE0', 'nczitzk'], - handler, - description: `| All | Announcements | Events | Safety & Alignment | Community | Product | Culture & Careers | Milestones | Research | -| --- | ------------- | ------ | ------------------ | --------- | ------- | ------------------- | ---------- | -------- | -| | announcements | events | safety-alignment | community | product | culture-and-careers | milestones | research |`, -}; - -async function handler(ctx) { - const tag = ctx.req.param('tag') || ''; - - const rootUrl = 'https://openai.com'; - const blogRootUrl = 'https://openai.com/blog'; - const blogOriginUrl = `${rootUrl}/blog${tag === '' ? '' : `?topics=${tag}`}`; - - const apiUrl = new URL('/api/v1/blog-details', await getApiUrl()); - - // Construct API query - apiUrl.searchParams.append('sort', '-publicationDate,-createdAt'); - apiUrl.searchParams.append('page[size]', '20'); - apiUrl.searchParams.append('page[number]', '1'); - apiUrl.searchParams.append('include', 'media,topics,authors'); - if (tag) { - apiUrl.searchParams.append('filter[topics][slugs][0]', tag); - } - - const response = await got({ - method: 'get', - url: apiUrl, - }); - - const list = response.data.data.filter((entry) => entry.type === 'blog-details'); - - const items = await Promise.all( - list.map((item) => { - const attributes = item.attributes; - return parseArticle(ctx, blogRootUrl, attributes); - }) - ); - - const title = `OpenAI Blog${tag ? ` - ${toTitleCase(tag)}` : ''}`; - - return { - title, - link: blogOriginUrl, - item: items, - }; -} diff --git a/lib/routes/openai/common.ts b/lib/routes/openai/common.ts index d7b7889b6..30723d385 100644 --- a/lib/routes/openai/common.ts +++ b/lib/routes/openai/common.ts @@ -1,8 +1,77 @@ import cache from '@/utils/cache'; import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; +import { DataItem } from '@/types'; import { load } from 'cheerio'; import { art } from '@/utils/render'; import path from 'node:path'; +import { config } from '@/config'; + +export const BASE_URL = new URL('https://openai.com'); + +/** Fetch the details of an article. */ +export const fetchArticleDetails = async (url: string) => { + const page = await ofetch(url); + const $ = load(page); + + const $article = $('#main article'); + + const categories = $('h1') + .prev() + .find('a[href]') + .toArray() + .map((element) => $(element).text()); + + // Article header (title, sub title and categories) + $($article.find('h1').parents().get(4)).remove(); + // Related articles (can be the #citations section in some cases, so the last child needs to be removed first) + $article.children().last().remove(); + // Article authors and tags + $article.find('#citations').remove(); + + return { + content: $article.html() ?? undefined, + // Categories can be found on https://openai.com/news/ and https://openai.com/research/index/ + categories, + image: $('meta[property="og:image"]').attr('content'), + }; +}; + +/** Fetch all articles from OpenAI's RSS feed. */ +export const fetchArticles = async (limit: number): Promise => { + const page = await ofetch('https://openai.com/news/rss.xml', { + responseType: 'text', + headers: { 'User-Agent': config.ua }, + }); + + const $ = load(page, { xml: true }); + + return Promise.all( + $('item') + .toArray() + .slice(0, limit) + .map>((element) => { + const id = $(element).find('guid').text(); + + return cache.tryGet(`openai:news:${id}`, async () => { + const title = $(element).find('title').text(); + const pubDate = $(element).find('pubDate').text(); + const link = $(element).find('link').text(); + + const { content, categories } = await fetchArticleDetails(link); + + return { + guid: id, + title, + link, + pubDate, + description: content, + category: categories, + } as DataItem; + }) as Promise; + }) + ); +}; const getApiUrl = async () => { const blogRootUrl = 'https://openai.com/blog'; diff --git a/lib/routes/openai/news.ts b/lib/routes/openai/news.ts new file mode 100644 index 000000000..2f399bdeb --- /dev/null +++ b/lib/routes/openai/news.ts @@ -0,0 +1,32 @@ +import { Route } from '@/types'; +import { fetchArticles, BASE_URL } from './common'; +import { Context } from 'hono'; + +export const route: Route = { + path: '/news', + categories: ['programming'], + example: '/openai/news', + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'News', + maintainers: ['goestav', 'StevenRCE0', 'nczitzk'], + handler, +}; + +async function handler(ctx: Context) { + const limit = Number.parseInt(ctx.req.query('limit') || '10'); + + const link = new URL('/news/', BASE_URL).href; + + return { + title: 'OpenAI News', + link, + item: await fetchArticles(limit), + }; +}