From d1fc409436cbcae711062f7d60d496d2239e98ca Mon Sep 17 00:00:00 2001 From: karasu Date: Sat, 18 Oct 2025 13:35:11 +0800 Subject: [PATCH] refactor(route): stdaily (#20313) * refactor(route): stdaily * fix: replace `sort` with `toSorted` --- lib/routes/stdaily/digitalpaper.ts | 214 ++++++++----------- lib/routes/stdaily/namespace.ts | 2 +- lib/routes/stdaily/templates/description.art | 12 +- lib/routes/stdaily/types.ts | 41 ++++ 4 files changed, 136 insertions(+), 133 deletions(-) create mode 100644 lib/routes/stdaily/types.ts diff --git a/lib/routes/stdaily/digitalpaper.ts b/lib/routes/stdaily/digitalpaper.ts index 131681aa6..4c33df794 100644 --- a/lib/routes/stdaily/digitalpaper.ts +++ b/lib/routes/stdaily/digitalpaper.ts @@ -1,146 +1,108 @@ -import { Route } from '@/types'; - +import type { Data, DataItem, Route } from '@/types'; +import ofetch from '@/utils/ofetch'; import cache from '@/utils/cache'; -import path from 'node:path'; +import { parseDate } from '@/utils/parse-date'; +import type { ArticleDetailResponse, ArticleListResponse, DateListResponse, SectionListResponse } from './types'; import { art } from '@/utils/render'; -import got from '@/utils/got'; +import path from 'node:path'; +import timezone from '@/utils/timezone'; +import dayjs from 'dayjs'; import { load } from 'cheerio'; -const renderDescription = ({ subtitle, quotation, pics, article }) => - art(path.join(__dirname, 'templates/description.art'), { - subtitle, - quotation, - pics, - article, - }); - -const getPageLength = async (url) => { - const response = await got({ - method: 'get', - url, - }); - - const $ = load(response.data); - const pageLength = $('.right1 .bmname').children().length; - return { pageLength, $ }; -}; - -const getArticleList = ($, paperUrl) => { - const pageName = $('.zi .zi-top .banci strong').text(); - const list = $('.zi-meat ul>li') - .toArray() - .map((item) => { - const link = $(item).find('a').attr('href'); - const title = $(item).find('a div').text(); - return { - link: `${paperUrl}/${link}`, - title: `[${pageName}] ${title}`, - // pubDate, - }; - }); - - return list; -}; -const createGetPageArticleList = (paperUrl, page = 1) => { - const getPageArticleList = async () => { - const currentUrl = `${paperUrl}/node_${page + 1}.htm`; - const response = await got({ - method: 'get', - url: currentUrl, - }); - - const $ = load(response.data); - return getArticleList($, paperUrl); - }; - - return getPageArticleList; -}; - -const getListArticles = async (list, cache) => { - const items = await Promise.all( - list.map((item) => - cache.tryGet(item.link, async () => { - const { data } = await got(item.link); - const $ = load(data); - - const quotation = $('.right-meat .yinti').text(); - const subtitle = $('.right-meat .futi').text(); - const article = $('.right-meat .tuwen .article #ozoom').html(); - const pics = $('.right-meat .tuwen .picture') - .toArray() - .map((item) => { - const pic = {}; - $(item) - .find('tr') - .toArray() - .map((row) => { - const src = $(row).find('img').attr('src'); - if (src) { - pic.src = src; - } else { - pic.des = $(row).find('td').text(); - } - return null; - }); - return pic; - }); - - item.author = $('.right-meat .author').text(); - item.description = renderDescription({ subtitle, quotation, article, pics }); - - return item; - }) - ) - ); - return items; -}; +const SITE_ID = '811c18b08cf04e79be3b67d6902ee1a7'; +const CODE = 'KJRB'; +const API_HOST = 'https://epaper.stdaily.com/stdailynewspaperapi'; export const route: Route = { path: '/digitalpaper', categories: ['traditional-media'], example: '/stdaily/digitalpaper', - parameters: {}, - features: { - requireConfig: false, - requirePuppeteer: false, - antiCrawler: false, - supportBT: false, - supportPodcast: false, - supportScihub: false, - }, name: '科技日报', - maintainers: ['lyqluis'], + maintainers: ['lyqluis', 'KarasuShin'], handler, + features: { + supportRadar: true, + }, + radar: [ + { + source: ['epaper.stdaily.com/statics/technology-site/index.html'], + target: '/digitalpaper', + }, + ], }; async function handler() { - const date = new Date(); - const dateStr = date.toLocaleDateString('zh-CN', { - year: 'numeric', - month: '2-digit', - day: '2-digit', + const dateListResponse = await ofetch(`${API_HOST}/uv/article/period/date`, { + method: 'POST', + body: { + code: CODE, + date: dayjs(new Date(), 'YYYY-MM-01'), + }, + headers: { 'X-Requested-With': 'XMLHttpRequest' }, }); - const formatedDate = dateStr.replace('/', '-'); // 'yyyy-mm/dd' - const rootUrl = 'http://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html'; - const currentUrl = `${rootUrl}/${formatedDate}`; + const lastDate = dateListResponse.obj.dateList.toSorted().at(-1); - // get page - const restPageLists = []; - const allPageArticleLists = []; - const { pageLength, $ } = await getPageLength(`${currentUrl}/node_2.htm`); - allPageArticleLists.push(...getArticleList($, currentUrl)); - for (let i = 2; i <= pageLength; i++) { - restPageLists.push(createGetPageArticleList(currentUrl, i)); - } - // get pages' article list - allPageArticleLists.push(...(await Promise.all(restPageLists.map((getPageArticleList) => getPageArticleList()))).flat()); - // get all articles - const items = await getListArticles(allPageArticleLists, cache); + const sectionListResponse = await ofetch(`${API_HOST}/uv/article/period/periodTime`, { + method: 'POST', + body: { + code: CODE, + periodTime: lastDate, + siteId: SITE_ID, + }, + headers: { 'X-Requested-With': 'XMLHttpRequest' }, + }); + + const items = await Promise.all( + sectionListResponse.obj.editionList.map((section) => + cache.tryGet(`stdaily:epaper:${section.id}`, async () => { + const { list } = await ofetch(`${API_HOST}/uv/article/article/editionId`, { + method: 'POST', + body: { + id: section.id, + siteId: SITE_ID, + }, + headers: { 'X-Requested-With': 'XMLHttpRequest' }, + }); + + return await Promise.all( + list.map( + async (article) => + await cache.tryGet(`stddaily:epaper:article:${article.id}`, async () => { + const { + obj: { articleVo }, + } = await ofetch(`${API_HOST}/uv/article/article/articleId`, { + method: 'POST', + body: { + code: CODE, + id: article.id, + siteId: SITE_ID, + }, + headers: { 'X-Requested-With': 'XMLHttpRequest' }, + }); + + return { + title: `${articleVo.editionName.match(/：(.*)/)?.[1]} - ${load(articleVo.title).text()}`, + description: art(path.join(__dirname, 'templates/description.art'), { + subtitle: articleVo.subtitle, + pics: articleVo.picList ?? [], + content: articleVo.content, + }), + pubDate: timezone(parseDate(articleVo.periodTime), +8), + author: articleVo.author, + link: `https://epaper.stdaily.com/statics/technology-site/index.html#/home?isDetail=1¤tNewsId=${article.id}¤tVersionName=${articleVo.editionName}¤tVersion=${Number(section.editionCode)}&timeValue=${articleVo.periodTime}`, + } as DataItem; + }) + ) + ); + }) + ) + ); return { - title: `科技日报`, - link: 'http://digitalpaper.stdaily.com', - item: items, - }; + title: '中国科技网 - 科技日报', + link: 'https://epaper.stdaily.com', + item: items.flat(), + image: 'https://www.stdaily.com/favicon.ico', + } as Data; } diff --git a/lib/routes/stdaily/namespace.ts b/lib/routes/stdaily/namespace.ts index 1975fcd4c..e38def8d2 100644 --- a/lib/routes/stdaily/namespace.ts +++ b/lib/routes/stdaily/namespace.ts @@ -2,6 +2,6 @@ import type { Namespace } from '@/types'; export const namespace: Namespace = { name: '中国科技网', - url: 'digitalpaper.stdaily.com', + url: 'epaper.stdaily.com', lang: 'zh-CN', }; diff --git a/lib/routes/stdaily/templates/description.art b/lib/routes/stdaily/templates/description.art index d12831395..50b5b3dfa 100644 --- a/lib/routes/stdaily/templates/description.art +++ b/lib/routes/stdaily/templates/description.art @@ -1,16 +1,16 @@ -{{if quotation}} -

{{quotation}}

-{{else if subtitle}} +{{if subtitle}}

{{subtitle}}

{{/if}} {{if pics.length}} {{each pics}}
- -

{{$value.des}}

+ + {{if $value.picText }} +

{{$value.picText}}

+ {{/if }}
{{/each}} {{/if}} -{{@article}} \ No newline at end of file +{{@content}} diff --git a/lib/routes/stdaily/types.ts b/lib/routes/stdaily/types.ts new file mode 100644 index 000000000..62cc81197 --- /dev/null +++ b/lib/routes/stdaily/types.ts @@ -0,0 +1,41 @@ +export interface DateListResponse { + obj: { + dateList: string[]; + }; +} + +export interface SectionListResponse { + obj: { + editionList: { + editionName: string; + periodId: string; + editionCode: string; + id: string; + }[]; + }; +} + +export interface ArticleListResponse { + list: { + id: string; + }[]; +} + +export interface ArticleDetailResponse { + obj: { + articleVo: { + title: string; + subtitle: string; + content: string; + editionName: string; + picList: + | null + | { + url: string; + picText: string; + }[]; + periodTime: string; + author: string; + }; + }; +}