From 8760e6bdad0a8d2c2d750df5a8349c861841cb0f Mon Sep 17 00:00:00 2001 From: Jerry K Jia Date: Wed, 22 Jun 2022 19:41:00 -0400 Subject: [PATCH] =?UTF-8?q?feat(route):=20=E5=BE=AE=E5=8D=9A+=E5=A4=B4?= =?UTF-8?q?=E6=9D=A1+=E7=9F=A5=E4=B9=8E=E7=83=AD=E6=90=9C=E7=9A=84?= =?UTF-8?q?=E5=85=B3=E9=94=AE=E8=AF=8D=E8=81=9A=E5=90=88=E8=BF=BD=E8=B8=AA?= =?UTF-8?q?=20(#9998)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/other.md | 12 ++ lib/v2/trending/allTrending.js | 169 ++++++++++++++++++++++++++ lib/v2/trending/maintainer.js | 3 + lib/v2/trending/radar.js | 35 ++++++ lib/v2/trending/router.js | 3 + lib/v2/trending/templates/content.art | 24 ++++ 6 files changed, 246 insertions(+) create mode 100644 lib/v2/trending/allTrending.js create mode 100644 lib/v2/trending/maintainer.js create mode 100644 lib/v2/trending/radar.js create mode 100644 lib/v2/trending/router.js create mode 100644 lib/v2/trending/templates/content.art diff --git a/docs/other.md b/docs/other.md index e5cbf9dc8..3c0cefd82 100644 --- a/docs/other.md +++ b/docs/other.md @@ -1026,3 +1026,15 @@ type 为 all 时，category 参数不支持 cost 和 free ### はてな匿名ダイアリー - 人気記事アーカイブ + +## 热搜聚合 + +### 关键词聚合追踪 + +追踪各大热搜榜上包含特定关键词的条目。 + +当前收录榜单：_微博热搜_、_今日头条热搜_、_知乎热搜_、_知乎热门视频_、_知乎热门话题_。 + +数据源: [trending-in-one](https://github.com/huqi-pr/trending-in-one) + + diff --git a/lib/v2/trending/allTrending.js b/lib/v2/trending/allTrending.js new file mode 100644 index 000000000..f0bb46e18 --- /dev/null +++ b/lib/v2/trending/allTrending.js @@ -0,0 +1,169 @@ +const dayjs = require('dayjs'); +dayjs.extend(require('dayjs/plugin/utc')); +dayjs.extend(require('dayjs/plugin/timezone')); +const got = require('@/utils/got'); +const { art } = require('@/utils/render'); +const path = require('path'); +const config = require('@/config').value; +const md5 = require('@/utils/md5'); + +// Constants +const DATA_REPO_BASE_URL = 'https://raw.githubusercontent.com/huqi-pr/trending-in-one/master/raw'; +const DATE_FORMAT = 'YYYY-MM-DD'; +// TODO: support custom data repo urls +const CHANNELS = { + 'toutiao-search': { + baseUrl: 'https://so.toutiao.com/search?keyword=', + name: '今日头条热搜', + }, + 'weibo-search': { + baseUrl: 'https://s.weibo.com/weibo?q=', + name: '微博热搜', + }, + 'zhihu-search': { + baseUrl: 'https://www.zhihu.com/search?q=', + name: '知乎热搜', + }, + 'zhihu-questions': { + baseUrl: 'https://www.zhihu.com/search?type=question&q=', + name: '知乎热门话题', + }, + 'zhihu-video': { + baseUrl: 'https://www.zhihu.com/search?type=video&q=', + name: '知乎热门视频', + }, +}; + +// Helper Functions +// TODO: integrate into CHANNELS +const processRawDataByChannel = { + 'toutiao-search': ({ word: title }) => ({ + // 源 url 存在 encoding 问题，暂时不使用 + url: CHANNELS['toutiao-search'].baseUrl + encodeURIComponent(title), + title, + }), + 'weibo-search': ({ title }) => ({ + // 源 url 存在 encoding 问题，暂时不使用 + url: CHANNELS['weibo-search'].baseUrl + encodeURIComponent(title), + title, + }), + 'zhihu-questions': (item) => item, + 'zhihu-search': ({ query }) => { + const title = query.trim(); + return { + // 源 url 存在 encoding 问题，暂时不使用 + url: CHANNELS['zhihu-search'].baseUrl + encodeURIComponent(title), + title, + }; + }, + 'zhihu-video': (item) => item, +}; + +const hasKeyword = (str, keywordList) => keywordList.some((keyword) => str.includes(keyword)); +const toShanghaiTimezone = (date) => dayjs.tz(date, 'Asia/Shanghai'); + +// Data Fetcher +// TODO: support channel selection +const fetchAllData = async (keywordList = [], dateList = [], cache) => { + const cachedGetData = (url, dataMappingFn) => + cache.tryGet( + url, + () => + got(url) + .json() + // Normalize data to { title, url } format + .then((res) => res.map((item) => dataMappingFn(item))), + config.cache.contentExpire, + false + ); + + const data = await Promise.all( + dateList.map(async (dateTime) => ({ + dateTime, + data: await Promise.all( + Object.keys(CHANNELS).map(async (channel) => ({ + name: CHANNELS[channel].name, + data: await cachedGetData(`${DATA_REPO_BASE_URL}/${channel}/${dateTime.format(DATE_FORMAT)}.json`, processRawDataByChannel[channel]).then((res) => res.filter(({ title }) => hasKeyword(title, keywordList))), + })) + ), + })) + ); + for (const i of data) { + i.count = i.data.reduce((acc, { data }) => acc + data.length, 0); + } + + return data.filter(({ count }) => count > 0); +}; + +// Generate Feed Items +const searchLinkUrls = (keyword) => [ + `https://tophub.today/search?e=tophub&q=${keyword}`, + `https://www.baidu.com/s?wd=${keyword}`, + `https://www.google.com/search?q=${keyword}`, + `https://www.zhihu.com/search?type=content&q=${keyword}`, + `https://s.weibo.com/weibo/${keyword}`, + `https://www.douyin.com/search/${keyword}`, + `https://so.toutiao.com/search?keyword=${keyword}`, +]; + +const searchLinkNames = ['热榜', '百度', '谷歌', '知乎', '微博', '抖音', '头条']; + +const createItem = ({ dateTime, data, count }, keywords, isToday) => { + const EOD = dateTime.endOf('day'); + const pubDate = isToday ? new Date() : EOD.toDate(); + return { + title: `${keywords.join(', ')} | ${dateTime.format(DATE_FORMAT)} 热点追踪 (${count})`, + author: 'Trending All In One', + pubDate, + description: art(path.join(__dirname, 'templates/content.art'), { + data, + queries: keywords.map((query) => ({ + links: searchLinkUrls(encodeURIComponent(query)).map((url, index) => `${searchLinkNames[index]}`), + key: query, + })), + }), + guid: `trending-all-in-one-${EOD.toISOString()}-${md5(JSON.stringify(data))}-${keywords.join('-')}`, + }; +}; + +// Main +module.exports = async (ctx) => { + // Prevent making over 100 requests per invocation + if (ctx.params.numberOfDays > 14) { + throw new Error('days must be less than 14'); + } + const numberOfDays = ctx.params.numberOfDays || 3; + const currentShanghaiDateTime = dayjs(toShanghaiTimezone(new Date())); + const currentShanghaiDateStr = currentShanghaiDateTime.format(DATE_FORMAT); + const dateList = []; + for (let i = 0; i < numberOfDays; i++) { + const d = currentShanghaiDateTime.subtract(i, 'day'); + dateList.push(d); + } + + const keywordList = ctx.params.keywords + .replace('，', ',') + .split(',') + .map((keyword) => keyword.trim()); + const keywordStr = keywordList.join(', '); + + const data = await fetchAllData(keywordList, dateList, ctx.cache); + const item = + data.length > 0 + ? data.map((i, index) => createItem(i, keywordList, index === 0)) + : [ + { + title: `${keywordStr} | ${currentShanghaiDateStr} 热点追踪 (0)`, + author: 'Trending All In One', + description: `近${numberOfDays}日的热搜都不包含下列关键词：${keywordStr}
请耐心等待，或添加更多关键词试试。`, + guid: `trending-all-in-one-${md5(JSON.stringify(data))}-${keywordList.join('-')}`, + }, + ]; + + ctx.state.data = { + title: `${keywordStr} | 热点聚合`, + description: `${keywordStr} | 今日头条热搜，知乎热门视频，知乎热搜榜，知乎热门话题，微博热搜榜聚合追踪`, + language: 'zh-cn', + item, + }; +}; diff --git a/lib/v2/trending/maintainer.js b/lib/v2/trending/maintainer.js new file mode 100644 index 000000000..1d256a559 --- /dev/null +++ b/lib/v2/trending/maintainer.js @@ -0,0 +1,3 @@ +module.exports = { + '/:keywords/:numberOfDays?': ['Jkker'], +}; diff --git a/lib/v2/trending/radar.js b/lib/v2/trending/radar.js new file mode 100644 index 000000000..697911c65 --- /dev/null +++ b/lib/v2/trending/radar.js @@ -0,0 +1,35 @@ +module.exports = { + 'toutiao.com': { + _name: '今日头条', + so: [ + { + title: '热搜关键词聚合追踪', + docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he', + source: ['/search'], + target: (params, url) => `/trending/${new URL(url).searchParams.get('keyword')}`, + }, + ], + }, + 'weibo.com': { + _name: '微博', + s: [ + { + title: '热搜关键词聚合追踪', + docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he', + source: '/weibo/:keyword', + target: (params) => `/trending/${params.keyword}}`, + }, + ], + }, + 'zhihu.com': { + _name: '知乎', + www: [ + { + title: '热搜关键词聚合追踪', + docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he', + source: ['/search'], + target: (params, url) => `/trending/${new URL(url).searchParams.get('q')}`, + }, + ], + }, +}; diff --git a/lib/v2/trending/router.js b/lib/v2/trending/router.js new file mode 100644 index 000000000..5dbb46808 --- /dev/null +++ b/lib/v2/trending/router.js @@ -0,0 +1,3 @@ +module.exports = (router) => { + router.get('/:keywords/:numberOfDays?', require('./allTrending')); +}; diff --git a/lib/v2/trending/templates/content.art b/lib/v2/trending/templates/content.art new file mode 100644 index 000000000..443d76b85 --- /dev/null +++ b/lib/v2/trending/templates/content.art @@ -0,0 +1,24 @@ +
+

热榜内容

+ {{each data channel}} + {{if channel.data.length !== 0}} +

{{channel.name}}

+ + {{/if}} + {{/each}} +

搜索关键词

+ {{each queries q}} +

{{q.key}}

+

+ {{each q.links l}} + {{@l}}   + {{/each}} +

+ {{/each}} +
\ No newline at end of file