diff --git a/docs/README.md b/docs/README.md index 621ddaa1f..c341526a4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -386,13 +386,13 @@ RSSHub 提供下列 API 接口: ### 简书 - + - + - + ### 知乎 diff --git a/docs/joinus/README.md b/docs/joinus/README.md index db05a0f40..e2d834e76 100644 --- a/docs/joinus/README.md +++ b/docs/joinus/README.md @@ -103,14 +103,82 @@ sidebar: auto ```js const $ = cheerio.load(data); // 使用 cheerio 加载返回的 HTML - const list = $('.note-list li'); // 使用 cheerio 选择器，选择 class="note-list" 下的所有
  • 元素，返回 cheerio node 对象数组 + const list = $('.note-list li').get(); + // 使用 cheerio 选择器，选择 class="note-list" 下的所有
  • 元素，返回 cheerio node 对象数组 + // cheerio get() 方法将 cheerio node 对象数组转换为 node 对象数组 // 注：每一个 cheerio node 对应一个 HTML DOM // 注：cheerio 选择器与 jquery 选择器几乎相同 // 参考 cheerio 文档：https://cheerio.js.org/ ``` - 赋值给 `ctx.state.data` + 使用 /jianshu/utils.js 类进行全文获取： + + ```js + const result = await util.ProcessFeed(list, ctx.cache); + ``` + + /jianshu/utils.js 类中的全文获取逻辑： + + ```js + const requestList = []; + + // 使用 Promise.all() 进行 async 并发 + const result = await Promise.all( + // 遍历每一篇文章 + list.map(async (item) => { + const $ = cheerio.load(item); + + // 还原相对链接为绝对链接 + const itemUrl = url.resolve(host, $('.title').attr('href')); + + // 查询缓存 + const cache = await caches.get(itemUrl); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + // 返回新的对象 + const single = { + title: $('.title').text(), + link: itemUrl, + author: $('.nickname').text(), + guid: itemUrl, + }; + + // 创建新的 axios 请求，存入 requestList 备用 + const es = axios.get(itemUrl); + requestList.push(es); + return Promise.resolve(single); + }) + ); + + // 并发所有 axios 请求 + const responses = await axios.all(requestList); + + // 循环 axios 结果 + for (let i = 0; i < responses.length; i++) { + const $ = cheerio.load(responses[i].data); + + // 根据网站 HTML，适配相应 selectors + result[i].description = $('.show-content-free').html(); + const date = new Date( + $('.publish-time') + .text() + .match(/\d{4}.\d{2}.\d{2} \d{2}:\d{2}/) + ); + + // 处理日期时区，东八区 GMT +8 即为 8 + const timeZone = 8; + const serverOffset = date.getTimezoneOffset() / 60; + result[i].pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString(); + + // 存入缓存，缓存时间（单位：秒）设置为 1（小时） * 60（分钟） * 60（秒）= 3600（秒） + caches.set(result[i].link, JSON.stringify(result[i]), 1 * 60 * 60); + } + ``` + + 将结果 `result` 赋值给 `ctx.state.data` ```js ctx.state.data = { @@ -118,25 +186,7 @@ sidebar: auto link: 'https://www.jianshu.com', // 选择 的 content 属性 description: $('meta[name="description"]').attr('content'), - item: - list && - list - .map((index, item) => { - // 遍历 cheerio node 对象数组 - // 注意，此处采用的为 cheerio map() 方法与 node 内置 map() 不同 - item = $(item); - return { - // 文章标题 - title: item.find('.title').text(), - // 文章正文，对每一个
  • DOM 进行操作，生成正文 - description: `作者：${item.find('.nickname').text()}
    描述：${item.find('.abstract').text()}
    `, - // 文章发布时间 - pubDate: new Date(item.find('.time').data('shared-at')).toUTCString(), - // 文章链接 - link: `https://www.jianshu.com${item.find('.title').attr('href')}`, - }; - }) - .get(), // cheerio get() 方法将 cheerio node 对象数组转换为 node 对象数组 + item: result, }; // 至此本路由结束 diff --git a/routes/bilibili/userChannel.js b/routes/bilibili/userChannel.js index 97d470bb3..5b250b679 100644 --- a/routes/bilibili/userChannel.js +++ b/routes/bilibili/userChannel.js @@ -7,7 +7,6 @@ module.exports = async (ctx) => { const userName = await cache.getUsernameFromUID(ctx, uid); const host = `https://api.bilibili.com/x/space/channel/video?mid=${uid}&cid=${cid}&pn=1&ps=10&order=0`; - const response = await axios({ method: 'get', url: host, diff --git a/routes/jianshu/collection.js b/routes/jianshu/collection.js index 35419053e..31a7b1ba9 100644 --- a/routes/jianshu/collection.js +++ b/routes/jianshu/collection.js @@ -1,5 +1,6 @@ const axios = require('../../utils/axios'); const cheerio = require('cheerio'); +const util = require('./utils'); module.exports = async (ctx) => { const id = ctx.params.id; @@ -15,24 +16,14 @@ module.exports = async (ctx) => { const data = response.data; const $ = cheerio.load(data); - const list = $('.note-list li'); + const list = $('.note-list li').get(); + + const result = await util.ProcessFeed(list, ctx.cache); ctx.state.data = { title: $('title').text(), link: `https://www.jianshu.com/c/${id}`, description: $('meta[name="description"]').attr('content') || $('title').text(), - item: - list && - list - .map((index, item) => { - item = $(item); - return { - title: item.find('.title').text(), - description: `作者：${item.find('.nickname').text()}
    描述：${item.find('.abstract').text()}
    `, - pubDate: new Date(item.find('.time').data('shared-at')).toUTCString(), - link: `https://www.jianshu.com${item.find('.title').attr('href')}`, - }; - }) - .get(), + item: result, }; }; diff --git a/routes/jianshu/home.js b/routes/jianshu/home.js index 7a4417355..8c0efa72d 100644 --- a/routes/jianshu/home.js +++ b/routes/jianshu/home.js @@ -1,5 +1,6 @@ const axios = require('../../utils/axios'); const cheerio = require('cheerio'); +const util = require('./utils'); module.exports = async (ctx) => { const response = await axios({ @@ -13,24 +14,14 @@ module.exports = async (ctx) => { const data = response.data; const $ = cheerio.load(data); - const list = $('.note-list li'); + const list = $('.note-list li').get(); + + const result = await util.ProcessFeed(list, ctx.cache); ctx.state.data = { title: '简书首页', link: 'https://www.jianshu.com', description: $('meta[name="description"]').attr('content'), - item: - list && - list - .map((index, item) => { - item = $(item); - return { - title: item.find('.title').text(), - description: `作者：${item.find('.nickname').text()}
    描述：${item.find('.abstract').text()}
    `, - pubDate: new Date(item.find('.time').data('shared-at')).toUTCString(), - link: `https://www.jianshu.com${item.find('.title').attr('href')}`, - }; - }) - .get(), + item: result, }; }; diff --git a/routes/jianshu/trending.js b/routes/jianshu/trending.js index b8cd7fd2c..4f8108aa6 100644 --- a/routes/jianshu/trending.js +++ b/routes/jianshu/trending.js @@ -1,9 +1,9 @@ const axios = require('../../utils/axios'); const cheerio = require('cheerio'); +const util = require('./utils'); module.exports = async (ctx) => { const link = `https://www.jianshu.com/trending/${ctx.params.timeframe}`; - const response = await axios({ method: 'get', url: link, @@ -15,24 +15,14 @@ module.exports = async (ctx) => { const data = response.data; const $ = cheerio.load(data); - const list = $('.note-list li'); + const list = $('.note-list li').get(); + + const result = await util.ProcessFeed(list, ctx.cache); ctx.state.data = { title: `简书 ${ctx.params.timeframe === 'weekly' ? '7' : '30'} 日热门`, link, description: `简书 ${ctx.params.timeframe === 'weekly' ? '7' : '30'} 日热门`, - item: - list && - list - .map((index, item) => { - item = $(item); - return { - title: item.find('.title').text(), - description: `作者：${item.find('.nickname').text()}
    描述：${item.find('.abstract').text()}
    `, - pubDate: new Date(item.find('.time').data('shared-at')).toUTCString(), - link: `https://www.jianshu.com${item.find('.title').attr('href')}`, - }; - }) - .get(), + item: result, }; }; diff --git a/routes/jianshu/user.js b/routes/jianshu/user.js index ad5e947a1..f5132b48e 100644 --- a/routes/jianshu/user.js +++ b/routes/jianshu/user.js @@ -1,5 +1,6 @@ const axios = require('../../utils/axios'); const cheerio = require('cheerio'); +const util = require('./utils'); module.exports = async (ctx) => { const id = ctx.params.id; @@ -15,24 +16,14 @@ module.exports = async (ctx) => { const data = response.data; const $ = cheerio.load(data); - const list = $('.note-list li'); + const list = $('.note-list li').get(); + + const result = await util.ProcessFeed(list, ctx.cache); ctx.state.data = { title: $('title').text(), link: `https://www.jianshu.com/u/${id}`, description: $('meta[name="description"]').attr('content') || $('title').text(), - item: - list && - list - .map((index, item) => { - item = $(item); - return { - title: item.find('.title').text(), - description: `作者：${item.find('.nickname').text()}
    描述：${item.find('.abstract').text()}
    `, - pubDate: new Date(item.find('.time').data('shared-at')).toUTCString(), - link: `https://www.jianshu.com${item.find('.title').attr('href')}`, - }; - }) - .get(), + item: result, }; }; diff --git a/routes/jianshu/utils.js b/routes/jianshu/utils.js new file mode 100644 index 000000000..f49efbbf8 --- /dev/null +++ b/routes/jianshu/utils.js @@ -0,0 +1,57 @@ +const axios = require('../../utils/axios'); +const cheerio = require('cheerio'); +const url = require('url'); + +const ProcessFeed = async (list, caches) => { + const host = 'https://www.jianshu.com'; + + const requestList = []; + + const result = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + + const itemUrl = url.resolve(host, $('.title').attr('href')); + + const cache = await caches.get(itemUrl); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const single = { + title: $('.title').text(), + link: itemUrl, + author: $('.nickname').text(), + guid: itemUrl, + }; + + const es = axios.get(itemUrl); + requestList.push(es); + return Promise.resolve(single); + }) + ); + + const responses = await axios.all(requestList); + + for (let i = 0; i < responses.length; i++) { + const $ = cheerio.load(responses[i].data); + + result[i].description = $('.show-content-free').html(); + + const date = new Date( + $('.publish-time') + .text() + .match(/\d{4}.\d{2}.\d{2} \d{2}:\d{2}/) + ); + + const timeZone = 8; + const serverOffset = date.getTimezoneOffset() / 60; + result[i].pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString(); + caches.set(result[i].link, JSON.stringify(result[i]), 1 * 60 * 60); + } + + return result; +}; + +module.exports = { + ProcessFeed, +};