From 0f51d2f93ab31695f082bae5f8157c2e77f2400d Mon Sep 17 00:00:00 2001 From: karasu Date: Tue, 5 Mar 2024 09:27:10 +0800 Subject: [PATCH] fix(route): cast (#14651) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(route): cast * fix(route): 兼容外链及视频页面特殊dom结构 * docs: 补充文档 * fix: use built in re-export * Update lib/routes/cast/index.ts * Update lib/routes/cast/index.ts * Update lib/routes/cast/index.ts * Update lib/routes/cast/index.ts * Update lib/routes/cast/index.ts * update maintainer --------- --- lib/routes/cast/index.ts | 117 +++++++++++++++-------------- lib/routes/cast/maintainer.ts | 2 +- lib/routes/cast/radar.ts | 4 +- lib/routes/cast/router.ts | 2 +- website/docs/routes/government.mdx | 16 +++- 5 files changed, 80 insertions(+), 61 deletions(-) diff --git a/lib/routes/cast/index.ts b/lib/routes/cast/index.ts index e722eeb6a..c01912a1e 100644 --- a/lib/routes/cast/index.ts +++ b/lib/routes/cast/index.ts @@ -1,71 +1,76 @@ -// @ts-nocheck import cache from '@/utils/cache'; -import got from '@/utils/got'; +import got, { type Response } from '@/utils/got'; import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import timezone from '@/utils/timezone'; const baseUrl = 'https://www.cast.org.cn'; -export default async (ctx) => { - const { column = 457 } = ctx.req.param(); - const { limit = 10 } = ctx.req.query(); - const link = `${baseUrl}/col/col${column}/index.html`; - const { data: response } = await got.post(`${baseUrl}/module/web/jpage/dataproxy.jsp`, { - searchParams: { - startrecord: 1, - endrecord: limit, - perpage: limit, - }, - form: { - col: 1, - appid: 1, - webid: 1, - path: '/', - columnid: column, - sourceContentType: 1, - unitid: 335, - webname: '中国科学技术协会', - permissiontype: 0, - }, - }); +interface ResponseData extends Response { + data: T; +} - const $ = load(response, { - xml: { - xmlMode: true, - }, - }); +async function parsePage(html: string) { + return await Promise.all( + load(html)('li') + .toArray() + .map((el) => { + const title = load(el)('a'); + let articleUrl = title.attr('href'); - const pageTitle = await cache.tryGet(link, async () => { - const { data: response } = await got(link); - const $ = load(response); - return $('head title').text(); - }); + if (articleUrl?.startsWith('http')) { + return { + title: title.text(), + link: title.attr('href'), + }; + } + articleUrl = `${baseUrl}${title.attr('href')}`; - const list = $('record') - .toArray() - .map((item) => { - item = load($(item).html(), null, false); - const a = item('a').first(); - return { - title: a.text(), - pubDate: parseDate(item('.list-data').text().trim(), 'DDYYYY/MM'), - link: `${baseUrl}${a.attr('href')}`, - }; - }); + return cache.tryGet(articleUrl, async () => { + const res = (await got.get(articleUrl!)) as ResponseData; + const article = load(res.data); + const pubDate = timezone(parseDate(article('meta[name=PubDate]').attr('content')!, 'YYYY-MM-DD HH:mm'), +8); - const items = await Promise.all( - list.map((item) => - cache.tryGet(item.link, async () => { - const { data: response } = await got(item.link); - const $ = load(response); - - item.description = $('#zoom').html(); - item.pubDate = timezone(parseDate($('meta[name=PubDate]').attr('content'), 'YYYY-MM-DD HH:mm'), +8); - - return item; + return { + title: title.text(), + pubDate, + description: article('#zoom').html(), + link: articleUrl, + }; + }); }) - ) ); +} + +export default async (ctx) => { + const { column, subColumn, category } = ctx.req.param(); + const { limit = 10 } = ctx.req.query(); + let link = `${baseUrl}/${column}/${subColumn}`; + if (category) { + link += `/${category}/index.html`; + } + const { data: indexData } = (await got.get(link)) as ResponseData; + + const $ = load(indexData); + + let items: any[] = []; + + // 新闻-视频首页特殊处理 + if (column === 'xw' && subColumn === 'SP' && !category) { + items = await parsePage(indexData); + } else { + const buildUnitScript = $('script[parseType="bulidstatic"]'); + const queryUrl = `${baseUrl}${buildUnitScript.attr('url')}`; + const queryData = JSON.parse(buildUnitScript.attr('querydata')?.replace(/'/g, '"') ?? '{}'); + queryData.paramJson = `{"pageNo":1,"pageSize":${limit}}`; + + const { data } = (await got.get(queryUrl, { + searchParams: new URLSearchParams(queryData), + })) as ResponseData<{ data: { html: string } }>; + + items = await parsePage(data.data.html); + } + + const pageTitle = $('head title').text(); ctx.set('data', { title: pageTitle, diff --git a/lib/routes/cast/maintainer.ts b/lib/routes/cast/maintainer.ts index 3cb4e108c..014bdb191 100644 --- a/lib/routes/cast/maintainer.ts +++ b/lib/routes/cast/maintainer.ts @@ -1,3 +1,3 @@ export default { - '/:column?': ['TonyRL'], + '/:column/:subColumn/:category?': ['KarasuShin', 'TonyRL'], }; diff --git a/lib/routes/cast/radar.ts b/lib/routes/cast/radar.ts index af8442645..2eb412c4d 100644 --- a/lib/routes/cast/radar.ts +++ b/lib/routes/cast/radar.ts @@ -5,8 +5,8 @@ export default { { title: '通用', docs: 'https://docs.rsshub.app/routes/government#zhong-guo-ke-xue-ji-shu-xie-hui', - source: ['/col/:column/index.html'], - target: (params) => `/cast/${params.column.replace('col', '')}`, + source: ['/:column/:subColumn/:category/index.html', '/:column/:subColumn/index.html'], + target: (params) => (params.category ? `/cast/${params.column}/${params.subColumn}/${params.category}` : `/cast/${params.column}/${params.subColumn}`), }, ], }, diff --git a/lib/routes/cast/router.ts b/lib/routes/cast/router.ts index 699c5cf13..a390ef458 100644 --- a/lib/routes/cast/router.ts +++ b/lib/routes/cast/router.ts @@ -1,3 +1,3 @@ export default (router) => { - router.get('/:column?', './index'); + router.get('/:column/:subColumn/:category?', './index'); }; diff --git a/website/docs/routes/government.mdx b/website/docs/routes/government.mdx index 71ca7b15e..36f8f842a 100644 --- a/website/docs/routes/government.mdx +++ b/website/docs/routes/government.mdx @@ -1348,7 +1348,21 @@ ### 通用 {#zhong-guo-ke-xue-ji-shu-xie-hui-tong-yong} - + + :::tip + 在路由末尾处加上 `?limit=限制获取数目` 来限制获取条目数量，默认值为`10` + ::: + + | 分类 | 编码 | + | -------- | ---- | + | 全景科协 | qjkx | + | 智库 | zk | + | 学术 | xs | + | 科普 | kp | + | 党建 | dj | + | 数据 | sj | + | 新闻 | xw | + ## 中国民用航空局 {#zhong-guo-min-yong-hang-kong-ju}