xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(route): add openai research support (#12484) 

* fix(route): split openai common logic to common.js

Signed-off-by: Guorui Yu <yuguorui@pku.edu.cn>

* fix(route): add openai research support

Signed-off-by: Guorui Yu <yuguorui@pku.edu.cn>

* fix(route): add openai research docs

Signed-off-by: Guorui Yu <yuguorui@pku.edu.cn>

---------

Signed-off-by: Guorui Yu <yuguorui@pku.edu.cn>
This commit is contained in:
Guorui Yu 2023-05-11 21:25:50 +08:00 committed by GitHub
parent c5ed15a42a
commit 0267e2c307
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 125 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/en/new-media.md
View File

@ -570,6 +570,10 @@ This route provides a flexible plan with full text content to subscribe specific
<RouteEn author="ETiV" example="/openai/chatgpt/release-notes" path="/openai/chatgpt/release-notes" />
### Research
<RouteEn author="yuguorui" example="/openai/research" path="/openai/research" />
## Phoronix
### News & Reviews

4
docs/new-media.md
View File

@ -1040,6 +1040,10 @@ IPFS 网关有可能失效，那时候换成其他网关。
<Route author="ETiV" example="/openai/chatgpt/release-notes" path="/openai/chatgpt/release-notes" />
### Research
<Route author="yuguorui" example="/openai/research" path="/openai/research" />
## OR
### 频道

59
lib/v2/openai/blog.js
View File

@ -1,8 +1,6 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { art } = require('@/utils/render');
const { toTitleCase } = require('@/utils/common-utils');
const path = require('path');
const { getApiUrl, parseArticle } = require('./common');
module.exports = async (ctx) => {
const tag = ctx.params.tag || '';
@ -11,18 +9,7 @@ module.exports = async (ctx) => {
const blogRootUrl = 'https://openai.com/blog';
const blogOriginUrl = `${rootUrl}/blog${tag === '' ? '' : `?topics=${tag}`}`;
// Find API base URL
const initResponse = await got({
method: 'get',
url: blogRootUrl,
});
let apiBaseUrl = initResponse.data
.toString()
.match(/(?<=TWILL_API_BASE:").+?(?=")/)[0]
.replaceAll('\\u002F', '/');
apiBaseUrl = apiBaseUrl + '/api/v1/blog-details';
const apiUrl = new URL(apiBaseUrl);
const apiUrl = new URL('/api/v1/blog-details', await getApiUrl());
// Construct API query
apiUrl.searchParams.append('sort', '-publicationDate,-createdAt');
@ -43,47 +30,7 @@ module.exports = async (ctx) => {
const items = await Promise.all(
list.map((item) => {
const attributes = item.attributes;
const textUrl = `${blogRootUrl}/${attributes.slug}`;
return ctx.cache.tryGet(attributes.slug, async () => {
const detailResponse = await got({
method: 'get',
url: textUrl,
});
let content = cheerio.load(detailResponse.data);
const authors = content('[aria-labelledby="metaAuthorsHeading"] > li > a > span > span')
.toArray()
.map((entry) => content(entry).text())
.join(', ');
// Leave out comments
const comments = content('*')
.contents()
.filter(function () {
return this.nodeType === 8;
});
comments.remove();
content = content('#content');
const imageSrc = attributes.seo.ogImageSrc;
const imageAlt = attributes.seo.ogImageAlt;
const article = art(path.join(__dirname, 'templates/article.art'), {
content,
imageSrc,
imageAlt,
});
return {
title: attributes.title,
author: authors,
description: article,
pubDate: attributes.createdAt,
category: attributes.tags.map((tag) => tag.title),
link: textUrl,
};
});
return parseArticle(ctx, blogRootUrl, attributes);
})
);

72
lib/v2/openai/common.js Normal file
View File

@ -0,0 +1,72 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { art } = require('@/utils/render');
const path = require('path');
const getApiUrl = async () => {
const blogRootUrl = 'https://openai.com/blog';
// Find API base URL
const initResponse = await got({
method: 'get',
url: blogRootUrl,
});
const apiBaseUrl = initResponse.data
.toString()
.match(/(?<=TWILL_API_BASE:").+?(?=")/)[0]
.replaceAll('\\u002F', '/');
return new URL(apiBaseUrl);
};
const parseArticle = (ctx, rootUrl, attributes) =>
ctx.cache.tryGet(attributes.slug, async () => {
const textUrl = `${rootUrl}/${attributes.slug}`;
const detailResponse = await got({
method: 'get',
url: textUrl,
});
let content = cheerio.load(detailResponse.data);
const authors = content('[aria-labelledby="metaAuthorsHeading"] > li > a > span > span')
.toArray()
.map((entry) => content(entry).text())
.join(', ');
// Leave out comments
const comments = content('*')
.contents()
.filter(function () {
return this.nodeType === 8;
});
comments.remove();
content = content('#content');
const imageSrc = attributes.seo.ogImageSrc;
const imageAlt = attributes.seo.ogImageAlt;
const article = art(path.join(__dirname, 'templates/article.art'), {
content,
imageSrc,
imageAlt,
});
// Not all article has tags
attributes.tags = attributes.tags || [];
return {
title: attributes.title,
author: authors,
description: article,
pubDate: attributes.createdAt,
category: attributes.tags.map((tag) => tag.title),
link: textUrl,
};
});
module.exports = {
getApiUrl,
parseArticle,
};

1
lib/v2/openai/maintainer.js
View File

@ -1,3 +1,4 @@
module.exports = {
'/blog/:tag?': ['StevenRCE0', 'nczitzk'],
'/research': ['yuguorui'],
};

8
lib/v2/openai/radar.js
View File

@ -23,5 +23,13 @@ module.exports = {
target: () => '/openai/chatgpt/release-notes',
},
],
research: [
{
title: 'Research',
docs: 'https://docs.rsshub.app/en/new-media.html#openai',
source: '/research',
target: () => '/openai/research',
},
],
},
};

32
lib/v2/openai/research.js Normal file
View File

@ -0,0 +1,32 @@
const got = require('@/utils/got');
const { getApiUrl, parseArticle } = require('./common');
module.exports = async (ctx) => {
const apiUrl = new URL('/api/v1/research-publications', await getApiUrl());
const researchRootUrl = 'https://openai.com/research';
// Construct API query
apiUrl.searchParams.append('sort', '-publicationDate,-createdAt');
apiUrl.searchParams.append('include', 'media');
const resp = await got({
method: 'get',
url: apiUrl,
});
const obj = resp.data;
const items = await Promise.all(
obj.data.map((item) => {
const attributes = item.attributes;
return parseArticle(ctx, researchRootUrl, attributes);
})
);
const title = 'OpenAI Research';
ctx.state.data = {
title,
link: researchRootUrl,
item: items,
};
};

1
lib/v2/openai/router.js
View File

@ -1,4 +1,5 @@
module.exports = function (router) {
router.get('/blog/:tag?', require('./blog'));
router.get('/chatgpt/release-notes', require('./chatgpt'));
router.get('/research', require('./research'));
};