From 25efba759638c9f9e41f0bbc90d8a2d670ff536d Mon Sep 17 00:00:00 2001 From: Tony Date: Tue, 26 Mar 2024 05:42:33 +0800 Subject: [PATCH] fix(route): guokr channel path (#14969) --- lib/routes/guokr/channel.ts | 58 ++++++++++++---------------------- lib/routes/guokr/scientific.ts | 40 +++++++---------------- lib/routes/guokr/utils.ts | 37 ++++++++++++++++++++++ 3 files changed, 68 insertions(+), 67 deletions(-) create mode 100644 lib/routes/guokr/utils.ts diff --git a/lib/routes/guokr/channel.ts b/lib/routes/guokr/channel.ts index 560308a1c..ee52d5de1 100644 --- a/lib/routes/guokr/channel.ts +++ b/lib/routes/guokr/channel.ts @@ -1,15 +1,6 @@ import { Route } from '@/types'; -import cache from '@/utils/cache'; import got from '@/utils/got'; - -async function loadFullPage(id) { - const link = `https://apis.guokr.com/minisite/article/${id}.json`; - const content = await cache.tryGet(link, async () => { - const res = await got(link); - return res.data.result.content; - }); - return content; -} +import { parseList, parseItem } from './utils'; const channelMap = { calendar: 'pac', @@ -19,21 +10,13 @@ const channelMap = { }; export const route: Route = { - path: '/:channel', + path: '/column/:channel', categories: ['new-media'], - example: '/guokr/calendar', + example: '/guokr/column/calendar', parameters: { channel: '专栏类别' }, - features: { - requireConfig: false, - requirePuppeteer: false, - antiCrawler: false, - supportBT: false, - supportPodcast: false, - supportScihub: false, - }, radar: [ { - source: ['guokr.com/'], + source: ['guokr.com/:channel'], }, ], name: '果壳网专栏', @@ -48,29 +31,28 @@ export const route: Route = { async function handler(ctx) { const channel = channelMap[ctx.req.param('channel')] ?? ctx.req.param('channel'); - const response = await got(`https://www.guokr.com/apis/minisite/article.json?retrieve_type=by_wx&channel_key=${channel}&offset=0&limit=10`); - const items = response.data.result; + const { data: response } = await got(`https://www.guokr.com/apis/minisite/article.json`, { + searchParams: { + retrieve_type: 'by_wx', + channel_key: channel, + offset: 0, + limit: 10, + }, + }); + const result = parseList(response.result); - if (items.length === 0) { + if (result.length === 0) { throw new Error('Unknown channel'); } - const channel_name = items[0].channels[0].name; - const channel_url = items[0].channels[0].url; + const channelName = result[0].channels[0].name; + const channelUrl = result[0].channels[0].url; - const result = await Promise.all( - items.map(async (item) => ({ - title: item.title, - description: await loadFullPage(item.id), // Mercury 无法正确解析全文，故这里手动加载 - pubDate: item.date_published, - link: item.url, - author: item.author.nickname, - })) - ); + const items = await Promise.all(result.map((item) => parseItem(item))); return { - title: `果壳网 ${channel_name}`, - link: channel_url, - item: result, + title: `果壳网 ${channelName}`, + link: channelUrl, + item: items, }; } diff --git a/lib/routes/guokr/scientific.ts b/lib/routes/guokr/scientific.ts index bafe9a68a..691f1cc4f 100644 --- a/lib/routes/guokr/scientific.ts +++ b/lib/routes/guokr/scientific.ts @@ -1,21 +1,11 @@ import { Route } from '@/types'; -import cache from '@/utils/cache'; import got from '@/utils/got'; -import { load } from 'cheerio'; +import { parseList, parseItem } from './utils'; export const route: Route = { path: '/scientific', categories: ['new-media'], example: '/guokr/scientific', - parameters: {}, - features: { - requireConfig: false, - requirePuppeteer: false, - antiCrawler: false, - supportBT: false, - supportPodcast: false, - supportScihub: false, - }, radar: [ { source: ['guokr.com/scientific', 'guokr.com/'], @@ -28,29 +18,21 @@ export const route: Route = { }; async function handler() { - const response = await got('https://www.guokr.com/apis/minisite/article.json?retrieve_type=by_subject&limit=20&offset=0'); + const { data: response } = await got('https://www.guokr.com/beta/proxy/science_api/articles', { + searchParams: { + retrieve_type: 'by_category', + page: 1, + }, + }); - const result = response.data.result; + const result = parseList(response); + + const items = await Promise.all(result.map((item) => parseItem(item))); return { title: '果壳网 科学人', link: 'https://www.guokr.com/scientific', description: '果壳网 科学人', - item: await Promise.all( - result.map((item) => - cache.tryGet(item.url, async () => { - const res = await got(item.url); - const $ = load(res.data); - item.description = $('.eflYNZ #js_content').css('visibility', 'visible').html() ?? $('.bxHoEL').html(); - return { - title: item.title, - description: item.description, - pubDate: item.date_published, - link: item.url, - author: item.author.nickname, - }; - }) - ) - ), + item: items, }; } diff --git a/lib/routes/guokr/utils.ts b/lib/routes/guokr/utils.ts new file mode 100644 index 000000000..8b0ed9d5f --- /dev/null +++ b/lib/routes/guokr/utils.ts @@ -0,0 +1,37 @@ +import { parseDate } from '@/utils/parse-date'; +import cache from '@/utils/cache'; +import got from '@/utils/got'; +import * as cheerio from 'cheerio'; + +export const parseList = (result) => + result.map((item) => ({ + title: item.title, + description: item.summary, + pubDate: parseDate(item.date_published), + link: `https://www.guokr.com/article/${item.id}/`, + author: item.author.nickname, + category: item.subject?.name, + id: item.id, + channels: item.channels, + })); + +export const parseItem = (item) => + cache.tryGet(item.link, async () => { + const { data: res } = await got(`https://apis.guokr.com/minisite/article/${item.id}.json`); + const $ = cheerio.load(res.result.content); + + $('#meta_content').remove(); + $('div').each((_, elem) => { + const $elem = $(elem); + $elem.attr('style', $elem.attr('style')?.replaceAll(/display:none;|visibility: hidden;/g, '')); + }); + $('img').each((_, elem) => { + const $elem = $(elem); + if ($elem.attr('data-src')) { + $elem.attr('src', $elem.attr('data-src')); + } + }); + item.description = $.html(); + + return item; + });