From 26fbfbfa8e881813d7c37bfbebaf8d2f743359bb Mon Sep 17 00:00:00 2001 From: Rongrong <15956627+Rongronggg9@users.noreply.github.com> Date: Tue, 5 Apr 2022 20:05:45 +0800 Subject: [PATCH] =?UTF-8?q?feat(route):=20add=20=E5=85=AC=E4=BC=97?= =?UTF-8?q?=E5=8F=B7=20360=20(#9457)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rongrong <15956627+Rongronggg9@users.noreply.github.com> --- docs/new-media.md | 30 ++++++++++++++++++ lib/v2/gzh360/category.js | 8 +++++ lib/v2/gzh360/gzh.js | 18 +++++++++++ lib/v2/gzh360/maintainer.js | 4 +++ lib/v2/gzh360/radar.js | 19 ++++++++++++ lib/v2/gzh360/router.js | 4 +++ lib/v2/gzh360/universal.js | 53 ++++++++++++++++++++++++++++++++ lib/v2/gzh360/utils.js | 61 +++++++++++++++++++++++++++++++++++++ 8 files changed, 197 insertions(+) create mode 100644 lib/v2/gzh360/category.js create mode 100644 lib/v2/gzh360/gzh.js create mode 100644 lib/v2/gzh360/maintainer.js create mode 100644 lib/v2/gzh360/radar.js create mode 100644 lib/v2/gzh360/router.js create mode 100644 lib/v2/gzh360/universal.js create mode 100644 lib/v2/gzh360/utils.js diff --git a/docs/new-media.md b/docs/new-media.md index bfc496e43..6a0d3c7d4 100644 --- a/docs/new-media.md +++ b/docs/new-media.md @@ -1855,6 +1855,32 @@ area 分区选项 +## 公众号 360 + +### 公众号 + + + +### 分类 + + + +| `id` | 分类 | | `id` | 分类 | +| -------------------------- | --- | - | -------------------------- | -- | +| | 首页 | | `5d357ae6e2eb992114a3d592` | 育儿 | +| `5d357964e2eb992114a3d588` | 热门 | | `5d357b00e2eb992114a3d593` | 旅游 | +| `5d3579a2e2eb992114a3d589` | 搞笑 | | `5d357b17e2eb992114a3d594` | 职场 | +| `5d3579b0e2eb992114a3d58a` | 健康 | | `5d357b34e2eb992114a3d595` | 美食 | +| `5d3579bae2eb992114a3d58b` | 私房话 | | `5d357b4ae2eb992114a3d596` | 历史 | +| `5d357a10e2eb992114a3d58c` | 八卦精 | | `5d357b60e2eb992114a3d597` | 教育 | +| `5d357a4ae2eb992114a3d58d` | 科技咖 | | `5d357b76e2eb992114a3d598` | 星座 | +| `5d357a72e2eb992114a3d58e` | 财经迷 | | `5d357b8de2eb992114a3d599` | 体育 | +| `5d357a8be2eb992114a3d58f` | 汽车控 | | `5d357b9be2eb992114a3d59a` | 军事 | +| `5d357aa1e2eb992114a3d590` | 生活家 | | `5d357bc2e2eb992114a3d59b` | 游戏 | +| `5d357ab6e2eb992114a3d591` | 时尚圈 | | `5d357bd4e2eb992114a3d59c` | 萌宠 | + + + ## 谷歌新闻 ### 新闻 @@ -3372,6 +3398,10 @@ column 为 third 时可选的 category: +### 公众号 (公众号 360 来源) + +见 [#公众号 360](#gong-zhong-hao-360) + ### 公众号栏目 (非推送 & 历史消息) diff --git a/lib/v2/gzh360/category.js b/lib/v2/gzh360/category.js new file mode 100644 index 000000000..ded39c8b2 --- /dev/null +++ b/lib/v2/gzh360/category.js @@ -0,0 +1,8 @@ +const universal = require('./universal'); + +module.exports = async (ctx) => { + const path = 'category'; + const id = ctx.params.id ?? ''; + const titleHeader = '公众号 360 - '; + await universal(ctx, path, id, titleHeader); +}; diff --git a/lib/v2/gzh360/gzh.js b/lib/v2/gzh360/gzh.js new file mode 100644 index 000000000..b82692420 --- /dev/null +++ b/lib/v2/gzh360/gzh.js @@ -0,0 +1,18 @@ +const { mpIdEncode, mpIdDecode } = require('./utils'); +const universal = require('./universal'); + +module.exports = async (ctx) => { + const path = 'gzh_articles'; + + let name = ctx.params.name; + let id = name; + + // in order to support RSSHub Radar, we need to accept both name and id + try { + name = mpIdDecode(name); + } catch { + id = mpIdEncode(name); + } + + await universal(ctx, path, id, '', name, true, true); +}; diff --git a/lib/v2/gzh360/maintainer.js b/lib/v2/gzh360/maintainer.js new file mode 100644 index 000000000..e9c4b613c --- /dev/null +++ b/lib/v2/gzh360/maintainer.js @@ -0,0 +1,4 @@ +module.exports = { + '/category/:id?': ['Rongronggg9'], + '/gzh/:name?': ['Rongronggg9'], +}; diff --git a/lib/v2/gzh360/radar.js b/lib/v2/gzh360/radar.js new file mode 100644 index 000000000..8af5d2f79 --- /dev/null +++ b/lib/v2/gzh360/radar.js @@ -0,0 +1,19 @@ +module.exports = { + 'gzh360.com': { + _name: '公众号360', + web: [ + { + title: '公众号', + docs: 'https://docs.rsshub.app/new-media.html#gong-zhong-hao-360', + source: ['/gzh_articles', '/gzh', '/'], + target: (params, url) => `/gzh360/gzh/${new URL(url).searchParams.get('id') ?? ''}`, + }, + { + title: '分类', + docs: 'https://docs.rsshub.app/new-media.html#gong-zhong-hao-360', + source: ['/category', '/'], + target: (params, url) => `/gzh360/category/${new URL(url).searchParams.get('id') ?? ''}`, + }, + ], + }, +}; diff --git a/lib/v2/gzh360/router.js b/lib/v2/gzh360/router.js new file mode 100644 index 000000000..358481fbe --- /dev/null +++ b/lib/v2/gzh360/router.js @@ -0,0 +1,4 @@ +module.exports = function (router) { + router.get('/category/:id?', require('./category')); + router.get('/gzh/:name', require('./gzh')); +}; diff --git a/lib/v2/gzh360/universal.js b/lib/v2/gzh360/universal.js new file mode 100644 index 000000000..c8bd7998b --- /dev/null +++ b/lib/v2/gzh360/universal.js @@ -0,0 +1,53 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const { finishArticleItem } = require('./utils'); +const config = require('@/config').value; + +const getInitEntry = async (url) => + await got(url) + .then((_r) => _r.data) + .catch((err) => { + if (err.response.statusCode === 404) { + throw new Error('This category / WeChat Official Account is not found on GZH360.'); + } + throw err; + }); + +module.exports = async (ctx, path, id, titleHeader = '', custom_title = null, skipAuthor = false, cacheInitEntry = false) => { + const rootUrl = 'http://web.gzh360.com'; + const currentUrl = `${rootUrl}/${id ? `${path}?id=${id}` : ''}`; + + const respData = cacheInitEntry ? await ctx.cache.tryGet(currentUrl, async () => await getInitEntry(currentUrl), config.cache.routeExpire, false) : await getInitEntry(currentUrl); + const $ = cheerio.load(respData); + + const title = id ? $('head > title').text().split(' - ', 1)[0] : '首页'; + + let items = $('div.content div.news_desc') + .map((_, item) => { + item = $(item); + const link = item.find('h3 > a'); + const href = link.attr('href'); + const title = link.text(); + const pubDate = item.find('span.datecss > span').attr('data-timestamp'); + let author; + if (!skipAuthor) { + author = item.find('span.fromcss > a').text(); // only some pages have this + } + return { + link: `${href.startsWith('http') ? '' : rootUrl}${href}`, + title, + pubDate: parseDate(pubDate), + author, + }; + }) + .get(); + + items = await Promise.all(items.map((item) => finishArticleItem(ctx, item, skipAuthor))); + + ctx.state.data = { + title: `${titleHeader}${custom_title ?? title}`, + link: currentUrl, + item: items.filter((item) => item), + }; +}; diff --git a/lib/v2/gzh360/utils.js b/lib/v2/gzh360/utils.js new file mode 100644 index 000000000..03f304bd5 --- /dev/null +++ b/lib/v2/gzh360/utils.js @@ -0,0 +1,61 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); + +const invalidIdError = new RangeError('Invalid id'); + +const mpIdEncode = (name) => { + const onePassed = Buffer.from(name).toString('base64'); // one-pass uses the standard base64 alphabet + // two-passed base64 is always url-safe (`7f+/` never appear), you can prove it by yourself + return Buffer.from(onePassed).toString('base64'); +}; + +const mpIdDecode = (id) => { + // verify that the decoded name can be a valid WeChat mp name: https://kf.qq.com/faq/120911VrYVrA141110r2MRJV.html + // "公众号名称/昵称可设置4-30个字符（1个汉字算2字符）": 4 ASCII characters => 12, 15 Chinese characters => 80 + // and the id is a valid two-passed base64 string (`7f+/` never appear) + if (id.length < 12 || id.length > 80 || id.length % 4 !== 0 || !/^[a-eg-zA-Z0-68-9]+={0,2}$/.test(id)) { + throw invalidIdError; + } + const deSecondPassed = Buffer.from(id, 'base64').toString(); + // verify that it is a valid base64 string using standard base64 alphabet + if (deSecondPassed.length % 4 !== 0 || !/^[a-zA-Z0-9+/]+={0,2}$/.test(deSecondPassed)) { + throw invalidIdError; + } + const deFirstPassed = Buffer.from(deSecondPassed, 'base64').toString(); + // "空格不可在最前或者最后，且空格不可连续" + if (deFirstPassed.length < 2 || deFirstPassed.startsWith(' ') || deFirstPassed.endsWith(' ') || deFirstPassed.includes(' ')) { + throw invalidIdError; + } + return deFirstPassed; +}; + +const finishArticleItem = async (ctx, item, skipAuthor = false) => { + // the website is slow and unstable, so we need to ignore errors to avoid breaking the whole route + // we can't cache the item instead of the webpage because we need to retry when the last request failed + const article = await ctx.cache.tryGet( + item.link, + async () => + await got(item.link) + .then((_r) => _r.data) + .catch(() => null) // it is safe do that in tryGet because a false value always lead to a cache miss + ); + + if (article) { + const $ = cheerio.load(article); + if (!skipAuthor) { + item.author = item.author || $('[id=webbdgzh]+a').text(); + } + item.title = item.title || $('div.desc > h1').text(); + item.pubDate = item.pubDate || parseDate($('div.desc span[data-timestamp]').attr('data-timestamp')); + item.description = $('div.rich_media_content').html(); // sometimes it is an empty string due to the website's fault + } + + return item; +}; + +module.exports = { + mpIdEncode, + mpIdDecode, + finishArticleItem, +};