fix(route/openai): 重构 research 路由，使用 RSS 源替代已失效的 TWILL API (#21309)
* fix(route/openai): 重构 research 路由，使用 RSS 源替代已失效的 TWILL API - 移除旧的 getApiUrl 和 parseArticle 函数 - 为 fetchArticles 添加 category 过滤参数 - 提取文章作者信息 - 清理未使用的依赖（got, raw, renderToString） * fix(route/openai): 使用 parseDate 解析 pubDate 日期字符串 * fix(route/openai): 使用重定向后的 URL 作为文章链接 使用 ofetch.raw() 获取响应，捕获重定向后的最终 URL， 避免输出中包含会被重定向的旧链接。 * fix(route/openai): 补全 URL 尾部斜杠以避免 301 重定向 在请求文章详情前为 URL 补上尾部斜杠，直接请求最终地址， 避免每篇文章都触发一次 301 重定向，减少不必要的 HTTP 请求。
This commit is contained in:
chesha1
GitHub
parent
7c1634128f
commit
1c5e4b3702
|
|
@ -1,19 +1,19 @@
|
|||
import { load } from 'cheerio';
|
||||
import { raw } from 'hono/html';
|
||||
import { renderToString } from 'hono/jsx/dom/server';
|
||||
|
||||
import { config } from '@/config';
|
||||
import type { DataItem } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import got from '@/utils/got';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
export const BASE_URL = new URL('https://openai.com');
|
||||
|
||||
/** Fetch the details of an article. */
|
||||
export const fetchArticleDetails = async (url: string) => {
|
||||
const page = await ofetch(url);
|
||||
const $ = load(page);
|
||||
// Ensure trailing slash to avoid 301 redirect
|
||||
const normalizedUrl = url.endsWith('/') ? url : `${url}/`;
|
||||
const html = await ofetch(normalizedUrl, { responseType: 'text' });
|
||||
const $ = load(html);
|
||||
|
||||
const $article = $('#main article');
|
||||
|
||||
|
|
@ -23,6 +23,10 @@ export const fetchArticleDetails = async (url: string) => {
|
|||
.toArray()
|
||||
.map((element) => $(element).text());
|
||||
|
||||
const authors = $('[data-testid="author-list"] a')
|
||||
.toArray()
|
||||
.map((element) => $(element).text());
|
||||
|
||||
// Article header (title, sub title and categories)
|
||||
$($article.find('h1').parents().get(4)).remove();
|
||||
// Related articles (can be the #citations section in some cases, so the last child needs to be removed first)
|
||||
|
|
@ -35,11 +39,13 @@ export const fetchArticleDetails = async (url: string) => {
|
|||
// Categories can be found on https://openai.com/news/ and https://openai.com/research/index/
|
||||
categories,
|
||||
image: $('meta[property="og:image"]').attr('content'),
|
||||
author: authors.join(', ') || undefined,
|
||||
link: normalizedUrl,
|
||||
};
|
||||
};
|
||||
|
||||
/** Fetch all articles from OpenAI's RSS feed. */
|
||||
export const fetchArticles = async (limit: number): Promise<DataItem[]> => {
|
||||
export const fetchArticles = async (limit: number, category?: string): Promise<DataItem[]> => {
|
||||
const page = await ofetch('https://openai.com/news/rss.xml', {
|
||||
responseType: 'text',
|
||||
headers: { 'User-Agent': config.ua },
|
||||
|
|
@ -47,95 +53,32 @@ export const fetchArticles = async (limit: number): Promise<DataItem[]> => {
|
|||
|
||||
const $ = load(page, { xml: true });
|
||||
|
||||
let items = $('item').toArray();
|
||||
if (category) {
|
||||
items = items.filter((element) => $(element).find('category').text() === category);
|
||||
}
|
||||
|
||||
return Promise.all(
|
||||
$('item')
|
||||
.toArray()
|
||||
.slice(0, limit)
|
||||
.map<Promise<DataItem>>((element) => {
|
||||
const id = $(element).find('guid').text();
|
||||
items.slice(0, limit).map<Promise<DataItem>>((element) => {
|
||||
const id = $(element).find('guid').text();
|
||||
|
||||
return cache.tryGet(`openai:news:${id}`, async () => {
|
||||
const title = $(element).find('title').text();
|
||||
const pubDate = $(element).find('pubDate').text();
|
||||
const link = $(element).find('link').text();
|
||||
return cache.tryGet(`openai:news:${id}`, async () => {
|
||||
const title = $(element).find('title').text();
|
||||
const pubDate = parseDate($(element).find('pubDate').text());
|
||||
const link = $(element).find('link').text();
|
||||
|
||||
const { content, categories } = await fetchArticleDetails(link);
|
||||
const { content, categories, author, link: articleLink } = await fetchArticleDetails(link);
|
||||
|
||||
return {
|
||||
guid: id,
|
||||
title,
|
||||
link,
|
||||
pubDate,
|
||||
description: content,
|
||||
category: categories,
|
||||
} as DataItem;
|
||||
}) as Promise<DataItem>;
|
||||
})
|
||||
return {
|
||||
guid: id,
|
||||
title,
|
||||
link: articleLink,
|
||||
pubDate,
|
||||
description: content,
|
||||
category: categories,
|
||||
author,
|
||||
} as DataItem;
|
||||
}) as Promise<DataItem>;
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
const getApiUrl = async () => {
|
||||
const blogRootUrl = 'https://openai.com/blog';
|
||||
|
||||
// Find API base URL
|
||||
const initResponse = await got({
|
||||
method: 'get',
|
||||
url: blogRootUrl,
|
||||
});
|
||||
|
||||
const apiBaseUrl = initResponse.data
|
||||
.toString()
|
||||
.match(/(?<=TWILL_API_BASE:").+?(?=")/)[0]
|
||||
.replaceAll(String.raw`\u002F`, '/');
|
||||
|
||||
return new URL(apiBaseUrl);
|
||||
};
|
||||
|
||||
const parseArticle = (ctx, rootUrl, attributes) =>
|
||||
cache.tryGet(attributes.slug, async () => {
|
||||
const textUrl = `${rootUrl}/${attributes.slug}`;
|
||||
const detailResponse = await got({
|
||||
method: 'get',
|
||||
url: textUrl,
|
||||
});
|
||||
let content = load(detailResponse.data);
|
||||
|
||||
const authors = content('[aria-labelledby="metaAuthorsHeading"] > li > a > span > span')
|
||||
.toArray()
|
||||
.map((entry) => content(entry).text())
|
||||
.join(', ');
|
||||
|
||||
// Leave out comments
|
||||
const comments = content('*')
|
||||
.contents()
|
||||
.filter(function () {
|
||||
return this.nodeType === 8;
|
||||
});
|
||||
comments.remove();
|
||||
|
||||
content = content('#content');
|
||||
|
||||
const imageSrc = attributes.seo.ogImageSrc;
|
||||
const imageAlt = attributes.seo.ogImageAlt;
|
||||
|
||||
const article = renderToString(
|
||||
<>
|
||||
<img src={imageSrc ?? ''} alt={imageAlt ?? ''} />
|
||||
{raw(content.toString())}
|
||||
</>
|
||||
);
|
||||
|
||||
// Not all article has tags
|
||||
attributes.tags = attributes.tags || [];
|
||||
|
||||
return {
|
||||
title: attributes.title,
|
||||
author: authors,
|
||||
description: article,
|
||||
pubDate: attributes.createdAt,
|
||||
category: attributes.tags.map((tag) => tag.title),
|
||||
link: textUrl,
|
||||
};
|
||||
});
|
||||
|
||||
export { getApiUrl, parseArticle };
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import type { Route } from '@/types';
|
||||
import got from '@/utils/got';
|
||||
import type { Context } from 'hono';
|
||||
|
||||
import { getApiUrl, parseArticle } from './common';
|
||||
import type { Route } from '@/types';
|
||||
|
||||
import { BASE_URL, fetchArticles } from './common';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/research',
|
||||
|
|
@ -17,36 +18,17 @@ export const route: Route = {
|
|||
supportScihub: false,
|
||||
},
|
||||
name: 'Research',
|
||||
maintainers: ['yuguorui'],
|
||||
maintainers: ['yuguorui', 'chesha1'],
|
||||
handler,
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
const apiUrl = new URL('/api/v1/research-publications', await getApiUrl());
|
||||
const researchRootUrl = 'https://openai.com/research';
|
||||
|
||||
// Construct API query
|
||||
apiUrl.searchParams.append('sort', '-publicationDate,-createdAt');
|
||||
apiUrl.searchParams.append('include', 'media');
|
||||
|
||||
const resp = await got({
|
||||
method: 'get',
|
||||
url: apiUrl,
|
||||
});
|
||||
const obj = resp.data;
|
||||
|
||||
const items = await Promise.all(
|
||||
obj.data.map((item) => {
|
||||
const attributes = item.attributes;
|
||||
return parseArticle(ctx, researchRootUrl, attributes);
|
||||
})
|
||||
);
|
||||
|
||||
const title = 'OpenAI Research';
|
||||
async function handler(ctx: Context) {
|
||||
const limit = Number.parseInt(ctx.req.query('limit') || '10');
|
||||
const link = new URL('/research/index', BASE_URL).href;
|
||||
|
||||
return {
|
||||
title,
|
||||
link: researchRootUrl,
|
||||
item: items,
|
||||
title: 'OpenAI Research',
|
||||
link,
|
||||
item: await fetchArticles(limit, 'Research'),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue