diff --git a/lib/routes/huggingface/blog-community.ts b/lib/routes/huggingface/blog-community.ts index 4e2bb4190..e0bc18436 100644 --- a/lib/routes/huggingface/blog-community.ts +++ b/lib/routes/huggingface/blog-community.ts @@ -1,8 +1,8 @@ import { load } from 'cheerio'; -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import cache from '@/utils/cache'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; export const route: Route = { @@ -38,18 +38,18 @@ export const route: Route = { url: 'huggingface.co/blog/community', }; -interface CommunityBlog { - classNames: string; - posts: Post[]; - pagination: Pagination; - sort: string; - showBackLink: boolean; -} - -interface Pagination { - numItemsPerPage: number; - numTotalItems: number; - pageIndex: number; +interface AuthorData { + _id?: string; + avatarUrl: string; + fullname: string; + name: string; + type: 'org' | 'user'; + isPro?: boolean; + isHf: boolean; + isHfAdmin: boolean; + isMod: boolean; + followerCount: number; + isEnterprise?: boolean; } interface Post { @@ -60,60 +60,45 @@ interface Post { numCoauthors: number; publishedAt: string; slug: string; - status: Status; + status: string; title: string; upvotes: number; + thumbnail?: string; } -interface AuthorData { - _id?: string; - avatarUrl: string; - fullname: string; - name: string; - type: Type; - isPro?: boolean; - isHf: boolean; - isHfAdmin: boolean; - isMod: boolean; - followerCount: number; - isEnterprise?: boolean; -} - -export enum Type { - Org = 'org', - User = 'user', -} - -export enum Status { - Published = 'published', +interface CommunityBlogApiResponse { + posts: Post[]; + pagination: { + numItemsPerPage: number; + numTotalItems: number; + pageIndex: number; + }; } async function handler(ctx) { const { sort = 'trending' } = ctx.req.param(); - const { body: response } = await got(`https://huggingface.co/blog/community?sort=${sort}`); - const $ = load(response); + const response = await ofetch(`https://huggingface.co/api/blog/community?sort=${sort}`); - const data = $('div[data-target="CommunityBlogsContainer"]').attr('data-props'); - if (!data) { - throw new Error('Failed to fetch data from Huggingface Community Articles'); - } - const articles: CommunityBlog = JSON.parse(data); - const lists = articles.posts.map((item) => ({ + const { posts } = response; + + const lists = posts.map((item) => ({ title: item.title, link: `https://huggingface.co/blog/${item.authorData.name}/${item.slug}`, pubDate: parseDate(item.publishedAt), - author: item.authorData.name, + author: item.authorData.fullname || item.authorData.name, + upvotes: item.upvotes, + image: item.thumbnail ? new URL(item.thumbnail, 'https://huggingface.co').toString() : undefined, })); - const items = await Promise.all( + const items: DataItem[] = await Promise.all( lists.map((item) => cache.tryGet(item.link, async () => { - const { body: response } = await got(item.link); + const response = await ofetch(item.link); const $ = load(response); - $('.mb-4, h1, .mb-6, .not-prose').remove(); + $('.mb-4, .mb-6, .not-prose, h1').remove(); return { ...item, - description: $('.blog-content').html(), + description: $('.blog-content').html() ?? undefined, }; }) ) diff --git a/lib/routes/huggingface/blog-zh.ts b/lib/routes/huggingface/blog-zh.ts index 7db3abc69..cb1db3f5a 100644 --- a/lib/routes/huggingface/blog-zh.ts +++ b/lib/routes/huggingface/blog-zh.ts @@ -1,9 +1,29 @@ import { load } from 'cheerio'; -import type { Route } from '@/types'; -import got from '@/utils/got'; +import type { DataItem, Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; +interface AuthorData { + fullname?: string; + name: string; +} + +interface BlogItem { + slug: string; + title: string; + publishedAt: string; + authorsData: AuthorData[]; + upvotes: number; + thumbnail?: string; + tags: string[]; +} + +interface BlogApiResponse { + allBlogs: BlogItem[]; +} + export const route: Route = { path: '/blog-zh', categories: ['programming'], @@ -29,29 +49,36 @@ export const route: Route = { }; async function handler() { - const { body: response } = await got('https://huggingface.co/blog/zh'); - const $ = load(response); + const response = await ofetch('https://huggingface.co/api/blog/zh'); - /** @type {Array<{blog: {local: string, title: string, author: string, thumbnail: string, date: string, tags: Array}, blogUrl: string, lang: 'zh', link: string}>} */ - const papers = $('div[data-target="BlogThumbnail"]') - .toArray() - .map((item) => { - const props = $(item).data('props'); - const link = $(item).find('a').attr('href'); - return { - ...props, - link, - }; - }); + const { allBlogs } = response; - const items = papers.map((item) => ({ - title: item.blog.title, - link: `https://huggingface.co${item.link}`, - category: item.blog.tags, - pubDate: parseDate(item.blog.publishedAt), - author: item.blog.author, + const lists = allBlogs.map((blog) => ({ + title: blog.title, + link: `https://huggingface.co/blog/zh/${blog.slug}`, + pubDate: parseDate(blog.publishedAt), + author: blog.authorsData.map((author) => ({ + name: author.fullname || author.name, + })), + upvotes: blog.upvotes, + image: blog.thumbnail ? new URL(blog.thumbnail, 'https://huggingface.co').toString() : undefined, + category: blog.tags, })); + const items: DataItem[] = await Promise.all( + lists.map((item) => + cache.tryGet(item.link, async () => { + const response = await ofetch(item.link); + const $ = load(response); + $('.mb-4, .mb-6, .not-prose, h1').remove(); + return { + ...item, + description: $('.blog-content').html() ?? undefined, + }; + }) + ) + ); + return { title: 'Huggingface 中文博客', link: 'https://huggingface.co/blog/zh', diff --git a/lib/routes/huggingface/blog.ts b/lib/routes/huggingface/blog.ts index 50fb15853..c8381ae82 100644 --- a/lib/routes/huggingface/blog.ts +++ b/lib/routes/huggingface/blog.ts @@ -1,9 +1,29 @@ import { load } from 'cheerio'; import type { DataItem, Route } from '@/types'; -import got from '@/utils/got'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; +interface AuthorData { + fullname?: string; + name: string; +} + +interface BlogItem { + slug: string; + title: string; + publishedAt: string; + authorsData: AuthorData[]; + upvotes: number; + thumbnail: string; + tags: string[]; +} + +interface BlogApiResponse { + allBlogs: BlogItem[]; +} + export const route: Route = { path: '/blog', categories: ['programming'], @@ -28,58 +48,37 @@ export const route: Route = { url: 'huggingface.co/blog', }; -interface Author { - user: string; - guest: boolean; - org?: string; -} - -interface Blog { - authors: Author[]; - canonical: boolean; - isUpvotedByUser: boolean; - publishedAt: string; - slug: string; - title: string; - upvotes: number; - thumbnail: string; - guest: boolean; -} - -interface BlogData { - blog: Blog; - blogUrl: string; - lang: string; - loggedInUser: string; -} - async function handler() { - const { body: response } = await got('https://huggingface.co/blog'); - const $ = load(response); + const response = await ofetch('https://huggingface.co/api/blog'); - /** @type {Array<{blog: {local: string, title: string, author: string, thumbnail: string, date: string, tags: Array}, blogUrl: string, lang: 'zh', link: string}>} */ - const papers = $('div[data-target="BlogThumbnail"]') - .toArray() - .map((item) => { - const props = $(item).data('props') as BlogData; - const link = $(item).find('a').attr('href'); - return { - ...props, - link, - }; - }); + const { allBlogs } = response; - const items: DataItem[] = papers.map((item) => ({ - title: item.blog.title, - link: `https://huggingface.co${item.link}`, - pubDate: parseDate(item.blog.publishedAt), - author: item.blog.authors.map((author) => ({ - name: author.user, + const lists = allBlogs.map((blog) => ({ + title: blog.title, + link: `https://huggingface.co/blog/${blog.slug}`, + pubDate: parseDate(blog.publishedAt), + author: blog.authorsData.map((author) => ({ + name: author.fullname || author.name, })), - upvotes: item.blog.upvotes, - image: new URL(item.blog.thumbnail, 'https://huggingface.co').toString(), + upvotes: blog.upvotes, + image: blog.thumbnail ? new URL(blog.thumbnail, 'https://huggingface.co').toString() : undefined, + category: blog.tags, })); + const items: DataItem[] = await Promise.all( + lists.map((item) => + cache.tryGet(item.link, async () => { + const response = await ofetch(item.link); + const $ = load(response); + $('.mb-4, .mb-6, .not-prose, h1').remove(); + return { + ...item, + description: $('.blog-content').html() ?? undefined, + }; + }) + ) + ); + return { title: 'Huggingface 英文博客', link: 'https://huggingface.co/blog', diff --git a/lib/routes/huggingface/daily-papers.ts b/lib/routes/huggingface/daily-papers.ts index 1a76ef271..4489c9eb1 100644 --- a/lib/routes/huggingface/daily-papers.ts +++ b/lib/routes/huggingface/daily-papers.ts @@ -66,7 +66,7 @@ async function handler(ctx) { const { body: response } = await got(url); const $ = load(response); - const papers = $('main > div[data-target="DailyPapers"]').data('props') as PapersData; + const papers = $('div[data-target="DailyPapers"]').data('props') as PapersData; const items = papers.dailyPapers .filter((item) => item.paper.upvotes >= voteFliter)