From 59447e4b89002a2582c95ce62b1ea5e88c0205aa Mon Sep 17 00:00:00 2001 From: dxmpalb <46418049+dxmpalb@users.noreply.github.com> Date: Mon, 25 Feb 2019 11:37:20 +0800 Subject: [PATCH] =?UTF-8?q?add=20=E6=87=82=E7=90=83=E5=B8=9D=E5=A4=B4?= =?UTF-8?q?=E6=9D=A1&=E4=B8=93=E9=A2=98=E6=96=B0=E9=97=BB=20optimize=20?= =?UTF-8?q?=E7=93=A6=E6=96=AF=E5=85=AC=E4=BC=97=E5=8F=B7=E5=8E=86=E5=8F=B2?= =?UTF-8?q?=E6=96=87=E7=AB=A0=20(#1612)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 详细解释一下增加瓦斯公众号历史文章的原因： * 目前的瓦斯公众号抓取（阅读器为Inoreader为例）：想要抓取某微信公众号的“日报”栏目时，利用瓦斯源，加上参数过滤，理想状态下是可以抓取到想要的内容的；但是，Inoreader刚添加订阅源时，很大可能订阅源是空的（即该公众号上一次所发布的文章中不含“日报”关键字），此时阅读器会把抓取间隔设置为比较大的值（假设为6小时）。而6小时内，公众号发布了两次内容，第一次含“日报”，第二次不含“日报”，这样的话，阅读器再次抓取时，订阅源仍然为空，抓取间隔继续加大……最终导致根本抓取不到想要的内容。（叙述可能有些混乱，实际测试中确实如此，测试使用之一是“央视新闻”公众号的“早啊！新闻来了”栏目，几天（不含周末）下来，并未抓到一篇文章，其他例子中情况类似） * 抓取历史文章的情况下：条数限制为20，通常情况下很少公众号一天能发布超过20篇文章，因此首次理论上能抓到几篇想要的文章（“日报”等），然后接下来阅读器抓取频率并不会降低（最终会稳定在栏目发布间隔这个时间）。实际测试中确实有比较好的效果，而且不加参数过滤的话也完全不受影响。 --- docs/README.md | 16 ++++++++ lib/router.js | 2 + lib/routes/dongqiudi/special.js | 64 +++++++++++++++++++++++++++++++ lib/routes/dongqiudi/top_news.js | 53 +++++++++++++++++++++++++ lib/routes/tencent/wechat/wasi.js | 17 ++++++-- 5 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 lib/routes/dongqiudi/special.js create mode 100644 lib/routes/dongqiudi/top_news.js diff --git a/docs/README.md b/docs/README.md index d53b964e3..e696a8815 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2711,6 +2711,22 @@ Example: `https://store.steampowered.com/search/?specials=1&term=atelier` 中的 ### 懂球帝 +::: tip 提示 + +可以通过头条新闻+参数过滤的形式获得早报、专题等内容。 + +::: + + + + + +| 新闻大爆炸 | 懂球帝十佳球 | 懂球帝本周 MVP | +| ---------- | ------------ | -------------- | +| 41 | 52 | 53 | + + + ::: tip 提示 diff --git a/lib/router.js b/lib/router.js index e2c0304f7..c3e7dc8ed 100644 --- a/lib/router.js +++ b/lib/router.js @@ -448,6 +448,8 @@ router.get('/dongqiudi/daily', require('./routes/dongqiudi/daily')); router.get('/dongqiudi/result/:team', require('./routes/dongqiudi/result')); router.get('/dongqiudi/team_news/:team', require('./routes/dongqiudi/team_news')); router.get('/dongqiudi/player_news/:id', require('./routes/dongqiudi/player_news')); +router.get('/dongqiudi/special/:id', require('./routes/dongqiudi/special')); +router.get('/dongqiudi/top_news', require('./routes/dongqiudi/top_news')); // 维基百科 Wikipedia router.get('/wikipedia/mainland', require('./routes/wikipedia/mainland')); diff --git a/lib/routes/dongqiudi/special.js b/lib/routes/dongqiudi/special.js new file mode 100644 index 000000000..c66cbd87b --- /dev/null +++ b/lib/routes/dongqiudi/special.js @@ -0,0 +1,64 @@ +const axios = require('../../utils/axios'); +const cheerio = require('cheerio'); +const utils = require('./utils'); +const date = require('../../utils/date'); + +module.exports = async (ctx) => { + const id = ctx.params.id; + const response = await axios.get(`https://www.dongqiudi.com/special/${id}`); + + const $ = cheerio.load(response.data); + + const host = 'https://www.dongqiudi.com'; + + const list = $('.detail.special ul li h3') + .slice(0, 5) + .get(); + + const proList = []; + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('a').text(); + const itemUrl = host + $('a').attr('href'); + + const cache = await ctx.cache.get(itemUrl); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + const single = { + title, + link: itemUrl, + }; + + const es = axios.get(itemUrl); + proList.push(es); + return Promise.resolve(single); + }) + ); + + const responses = await axios.all(proList); + for (let i = 0; i < proList.length; i++) { + const $ = utils.ProcessVideo(cheerio.load(responses[i].data)); + const full = $('div.detail'); + + out[i].description = utils.ProcessHref(full.find('div:nth-of-type(1)')).html(); + out[i].author = full.find('span.name').text(); + out[i].pubDate = date( + full + .find('span.time') + .text() + .trim(), + 8 + ); + + ctx.cache.set(out[i].itemUrl, JSON.stringify(out[i]), 24 * 60 * 60); + } + ctx.state.data = { + title: `懂球帝专题-${id}`, + link: `https://www.dongqiudi.com/special/${id}`, + item: out.filter((e) => e !== undefined), + }; +}; diff --git a/lib/routes/dongqiudi/top_news.js b/lib/routes/dongqiudi/top_news.js new file mode 100644 index 000000000..855542f45 --- /dev/null +++ b/lib/routes/dongqiudi/top_news.js @@ -0,0 +1,53 @@ +const axios = require('../../utils/axios'); +const cheerio = require('cheerio'); +const utils = require('./utils'); +const date = require('../../utils/date'); + +module.exports = async (ctx) => { + const response = await axios.get('https://api.dongqiudi.com/app/tabs/iphone/1.json?mark=gif&version=576'); + const data = response.data.articles; + + const proList = []; + + const out = await Promise.all( + data.map(async (item) => { + const title = item.title; + const itemUrl = item.share; + + const cache = await ctx.cache.get(itemUrl); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + const single = { + title, + link: itemUrl, + }; + + const es = axios.get(itemUrl); + proList.push(es); + return Promise.resolve(single); + }) + ); + + const responses = await axios.all(proList); + for (let i = 0; i < proList.length; i++) { + const $ = utils.ProcessVideo(cheerio.load(responses[i].data)); + const full = $('div.detail'); + + out[i].description = utils.ProcessHref(full.find('div:nth-of-type(1)')).html(); + out[i].author = full.find('span.name').text(); + out[i].pubDate = date( + full + .find('span.time') + .text() + .trim(), + 8 + ); + } + ctx.state.data = { + title: '懂球帝头条新闻', + link: 'http://dongqiudi.com/', + item: out.filter((e) => e !== undefined), + }; +}; diff --git a/lib/routes/tencent/wechat/wasi.js b/lib/routes/tencent/wechat/wasi.js index ded6b2209..50f2c3019 100644 --- a/lib/routes/tencent/wechat/wasi.js +++ b/lib/routes/tencent/wechat/wasi.js @@ -10,13 +10,22 @@ module.exports = async (ctx) => { Referer: `https://w.qnmlgb.tech/authors/${id}/`, }, }); - const data = response.data.result.articles[0]; + const rawData = response.data.result.articles; + + const data = []; + + for (let i = 0; i < rawData.length; i++) { + const subArticles = rawData[i].sub_articles; + for (let j = 0; j < subArticles.length; j++) { + data.push(subArticles[j]); + } + } ctx.state.data = { - title: `${data.author.nickname}微信公众号`, + title: `${rawData[0].author.nickname}微信公众号`, link: `https://w.qnmlgb.tech/authors/${id}/`, - description: data.author.profile_desc, - item: data.sub_articles.map((item) => ({ + description: rawData[0].author.profile_desc, + item: data.map((item) => ({ title: item.article.title, description: `${item.article.digest}`, pubDate: new Date(item.article.datetime * 1000).toUTCString(),