* fix(route/openai): fetch news through RSS feed * refactor: rename blog route to news * refactor(route/openai): ensure limit parameter works as expected * refactor(route/openai): use user agent from configuration * refactor(route/openai): cache data items * refactor(route/openai): cache individual data items * refactor: read query string parameter instead of path parameter
140 lines
4.2 KiB
TypeScript
140 lines
4.2 KiB
TypeScript
import cache from '@/utils/cache';
|
|
import got from '@/utils/got';
|
|
import ofetch from '@/utils/ofetch';
|
|
import { DataItem } from '@/types';
|
|
import { load } from 'cheerio';
|
|
import { art } from '@/utils/render';
|
|
import path from 'node:path';
|
|
import { config } from '@/config';
|
|
|
|
export const BASE_URL = new URL('https://openai.com');
|
|
|
|
/** Fetch the details of an article. */
|
|
export const fetchArticleDetails = async (url: string) => {
|
|
const page = await ofetch(url);
|
|
const $ = load(page);
|
|
|
|
const $article = $('#main article');
|
|
|
|
const categories = $('h1')
|
|
.prev()
|
|
.find('a[href]')
|
|
.toArray()
|
|
.map((element) => $(element).text());
|
|
|
|
// Article header (title, sub title and categories)
|
|
$($article.find('h1').parents().get(4)).remove();
|
|
// Related articles (can be the #citations section in some cases, so the last child needs to be removed first)
|
|
$article.children().last().remove();
|
|
// Article authors and tags
|
|
$article.find('#citations').remove();
|
|
|
|
return {
|
|
content: $article.html() ?? undefined,
|
|
// Categories can be found on https://openai.com/news/ and https://openai.com/research/index/
|
|
categories,
|
|
image: $('meta[property="og:image"]').attr('content'),
|
|
};
|
|
};
|
|
|
|
/** Fetch all articles from OpenAI's RSS feed. */
|
|
export const fetchArticles = async (limit: number): Promise<DataItem[]> => {
|
|
const page = await ofetch('https://openai.com/news/rss.xml', {
|
|
responseType: 'text',
|
|
headers: { 'User-Agent': config.ua },
|
|
});
|
|
|
|
const $ = load(page, { xml: true });
|
|
|
|
return Promise.all(
|
|
$('item')
|
|
.toArray()
|
|
.slice(0, limit)
|
|
.map<Promise<DataItem>>((element) => {
|
|
const id = $(element).find('guid').text();
|
|
|
|
return cache.tryGet(`openai:news:${id}`, async () => {
|
|
const title = $(element).find('title').text();
|
|
const pubDate = $(element).find('pubDate').text();
|
|
const link = $(element).find('link').text();
|
|
|
|
const { content, categories } = await fetchArticleDetails(link);
|
|
|
|
return {
|
|
guid: id,
|
|
title,
|
|
link,
|
|
pubDate,
|
|
description: content,
|
|
category: categories,
|
|
} as DataItem;
|
|
}) as Promise<DataItem>;
|
|
})
|
|
);
|
|
};
|
|
|
|
const getApiUrl = async () => {
|
|
const blogRootUrl = 'https://openai.com/blog';
|
|
|
|
// Find API base URL
|
|
const initResponse = await got({
|
|
method: 'get',
|
|
url: blogRootUrl,
|
|
});
|
|
|
|
const apiBaseUrl = initResponse.data
|
|
.toString()
|
|
.match(/(?<=TWILL_API_BASE:").+?(?=")/)[0]
|
|
.replaceAll(String.raw`\u002F`, '/');
|
|
|
|
return new URL(apiBaseUrl);
|
|
};
|
|
|
|
const parseArticle = (ctx, rootUrl, attributes) =>
|
|
cache.tryGet(attributes.slug, async () => {
|
|
const textUrl = `${rootUrl}/${attributes.slug}`;
|
|
const detailResponse = await got({
|
|
method: 'get',
|
|
url: textUrl,
|
|
});
|
|
let content = load(detailResponse.data);
|
|
|
|
const authors = content('[aria-labelledby="metaAuthorsHeading"] > li > a > span > span')
|
|
.toArray()
|
|
.map((entry) => content(entry).text())
|
|
.join(', ');
|
|
|
|
// Leave out comments
|
|
const comments = content('*')
|
|
.contents()
|
|
.filter(function () {
|
|
return this.nodeType === 8;
|
|
});
|
|
comments.remove();
|
|
|
|
content = content('#content');
|
|
|
|
const imageSrc = attributes.seo.ogImageSrc;
|
|
const imageAlt = attributes.seo.ogImageAlt;
|
|
|
|
const article = art(path.join(__dirname, 'templates/article.art'), {
|
|
content,
|
|
imageSrc,
|
|
imageAlt,
|
|
});
|
|
|
|
// Not all article has tags
|
|
attributes.tags = attributes.tags || [];
|
|
|
|
return {
|
|
title: attributes.title,
|
|
author: authors,
|
|
description: article,
|
|
pubDate: attributes.createdAt,
|
|
category: attributes.tags.map((tag) => tag.title),
|
|
link: textUrl,
|
|
};
|
|
});
|
|
|
|
export { getApiUrl, parseArticle };
|