fix(route/yystv): fix cache collisions (#18739)

* fix(route/yystv): fix cache collisions induced by these two implementations

* chore(route/yystv): prefer existing types over new types

* fix(route/yystv): prefer caching processed items over raw web pages

* docs: fix maintainer github id

---------
This commit is contained in:
Jiahao Lee 2025-04-02 03:10:56 +08:00 committed by GitHub
parent b9a71029f3
commit 8c16cf41a7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 66 additions and 60 deletions

View File

@ -1,6 +1,6 @@
import { Route } from '@/types';
import cache from '@/utils/cache';
import got from '@/utils/got';
import { DataItem, Route } from '@/types';
import ofetch from '@/utils/ofetch';
import { fetchDataItemCached } from './fetcher';
import { load } from 'cheerio';
import { parseDate, parseRelativeDate } from '@/utils/parse-date';
@ -18,67 +18,65 @@ export const route: Route = {
supportScihub: false,
},
name: '游研社 - 分类文章',
maintainers: ['LightStrawberry'],
maintainers: ['betta-cyber', 'dousha'],
handler,
description: `| 推游 | 游戏史 | 大事件 | 文化 | 趣闻 | 经典回顾 |
| --------- | ------- | ------ | ------- | ---- | -------- |
| recommend | history | big | culture | news | retro |`,
description: `| 推游 | 游戏史 | 大事件 | 文化 | 趣闻 | 经典回顾 | 业界 |
| --------- | ------- | ------ | ------- | ---- | -------- | -------- |
| recommend | history | big | culture | news | retro | industry |`,
};
type ArticleEntry = DataItem & { link: string };
function getDescription(items: ArticleEntry[]): Promise<DataItem[]> {
return Promise.all(
items.map((item) =>
fetchDataItemCached(item.link, (pageContent) => {
const $ = load(pageContent);
const articleContent = $('.doc-content.rel').html() || '';
const assembledItem: DataItem = { ...item, description: articleContent };
return assembledItem;
})
)
);
}
async function handler(ctx) {
const category = ctx.req.param('category');
const url = `https://www.yystv.cn/b/${category}`;
const response = await got({
method: 'get',
url,
});
const response = await ofetch(url);
const $ = load(response);
const data = response.data;
const $ = load(data);
const first_part = $('.b-list-main-item')
.slice(0, 2)
.map(function () {
const info = {
title: $(this).find('.b-main-info-title').text(),
link: 'https://www.yystv.cn' + $(this).find('.b-main-info-title a').attr('href'),
pubDate: parseRelativeDate($(this).find('.b-main-createtime').text()),
author: $(this).find('.b-author').text(),
const firstPart = $('.b-list-main-item')
.toArray()
.map((element) => {
const s = $(element);
const info: ArticleEntry = {
title: s.find('.b-main-info-title').text(),
link: 'https://www.yystv.cn' + s.find('.b-main-info-title a').attr('href'),
pubDate: parseRelativeDate(s.find('.b-main-createtime').text()),
author: s.find('.b-author').text(),
};
return info;
})
.get();
});
const second_part = $('.list-container li')
.slice(0, 18)
.map(function () {
const info = {
title: $('.list-article-title', this).text(),
link: 'https://www.yystv.cn' + $('a', this).attr('href'),
pubDate: $('.c-999', this).text().includes('-') ? parseDate($('.c-999', this).text()) : parseRelativeDate($('.c-999', this).text()),
author: $('.handler-author-link', this).text(),
const secondPart = $('.list-container li')
.toArray()
.map((element) => {
const s = $(element);
const articleDate = s.find('.c-999').text();
const info: ArticleEntry = {
title: s.find('.list-article-title').text(),
link: 'https://www.yystv.cn' + s.find('a').attr('href'),
pubDate: articleDate.includes('-') ? parseDate(articleDate) : parseRelativeDate(articleDate),
author: s.find('.handler-author-link').text(),
};
return info;
})
.get();
});
const items = [...first_part, ...second_part];
function getDescription(items) {
return Promise.all(
items.map(async (currentValue) => {
currentValue.description = await cache.tryGet(currentValue.link, async () => {
const r = await got({
url: currentValue.link,
method: 'get',
});
const $ = load(r.data);
return $('.doc-content.rel').html();
});
return currentValue;
})
);
}
await getDescription(items).then(() => ({
const entries = [...firstPart, ...secondPart];
return await getDescription(entries).then((items) => ({
title: '游研社-' + $('title').text(),
link: `https://www.yystv.cn/b/${category}`,
item: items,

View File

@ -2,7 +2,7 @@ import type { Route, DataItem } from '@/types';
import ofetch from '@/utils/ofetch';
import { load } from 'cheerio';
import { parseRelativeDate } from '@/utils/parse-date';
import cache from '@/utils/cache';
import { fetchDataItemCached } from './fetcher';
export const route: Route = {
path: '/docs',
@ -49,14 +49,12 @@ async function handler() {
}) satisfies DataItem[];
const items = (await Promise.all(
itemList.map(
(item) =>
cache.tryGet(item.link, async () => {
const resp = await ofetch(item.link);
const $ = load(resp);
item.description = $('#main section.article-section .doc-content > div').html() || item.description;
return item;
}) as Promise<DataItem>
itemList.map((item) =>
fetchDataItemCached(item.link, (articleContent) => {
const $ = load(articleContent);
item.description = $('#main section.article-section .doc-content > div').html() || item.description;
return item;
})
)
)) satisfies DataItem[];

View File

@ -0,0 +1,10 @@
import { DataItem } from '@/types';
import cache from '@/utils/cache';
import ofetch from '@/utils/ofetch';
export function fetchDataItemCached(link: string, processor: (articleContent: string) => DataItem): Promise<DataItem> {
return cache.tryGet(link, async () => {
const page = await ofetch(link);
return processor(page);
}) as Promise<DataItem>;
}