add 懂球帝头条&专题新闻 optimize 瓦斯公众号历史文章 (#1612)
详细解释一下增加瓦斯公众号历史文章的原因： * 目前的瓦斯公众号抓取（阅读器为Inoreader为例）：想要抓取某微信公众号的“日报”栏目时，利用瓦斯源，加上参数过滤，理想状态下是可以抓取到想要的内容的；但是，Inoreader刚添加订阅源时，很大可能订阅源是空的（即该公众号上一次所发布的文章中不含“日报”关键字），此时阅读器会把抓取间隔设置为比较大的值（假设为6小时）。而6小时内，公众号发布了两次内容，第一次含“日报”，第二次不含“日报”，这样的话，阅读器再次抓取时，订阅源仍然为空，抓取间隔继续加大……最终导致根本抓取不到想要的内容。（叙述可能有些混乱，实际测试中确实如此，测试使用之一是“央视新闻”公众号的“早啊！新闻来了”栏目，几天（不含周末）下来，并未抓到一篇文章，其他例子中情况类似） * 抓取历史文章的情况下：条数限制为20，通常情况下很少公众号一天能发布超过20篇文章，因此首次理论上能抓到几篇想要的文章（“日报”等），然后接下来阅读器抓取频率并不会降低（最终会稳定在栏目发布间隔这个时间）。实际测试中确实有比较好的效果，而且不加参数过滤的话也完全不受影响。
This commit is contained in:
dxmpalb
DIYgod
parent
8434a3abe0
commit
59447e4b89
|
|
@ -2711,6 +2711,22 @@ Example: `https://store.steampowered.com/search/?specials=1&term=atelier` 中的
|
|||
|
||||
### 懂球帝
|
||||
|
||||
::: tip 提示
|
||||
|
||||
可以通过头条新闻+参数过滤的形式获得早报、专题等内容。
|
||||
|
||||
:::
|
||||
|
||||
<route name="头条新闻" author="dxmpalb" example="/dongqiudi/top_news" path="/dongqiudi/top_news"/>
|
||||
|
||||
<route name="专题" author="dxmpalb" example="/dongqiudi/special/41" path="/dongqiudi/special/:id" :paramsDesc="['专题 id, 可自行通过 https://www.dongqiudi.com/special/+数字匹配']">
|
||||
|
||||
| 新闻大爆炸 | 懂球帝十佳球 | 懂球帝本周 MVP |
|
||||
| ---------- | ------------ | -------------- |
|
||||
| 41 | 52 | 53 |
|
||||
|
||||
</route>
|
||||
|
||||
<route name="早报" author="HenryQW" example="/dongqiudi/daily" path="/dongqiudi/daily"/>
|
||||
|
||||
::: tip 提示
|
||||
|
|
|
|||
|
|
@ -448,6 +448,8 @@ router.get('/dongqiudi/daily', require('./routes/dongqiudi/daily'));
|
|||
router.get('/dongqiudi/result/:team', require('./routes/dongqiudi/result'));
|
||||
router.get('/dongqiudi/team_news/:team', require('./routes/dongqiudi/team_news'));
|
||||
router.get('/dongqiudi/player_news/:id', require('./routes/dongqiudi/player_news'));
|
||||
router.get('/dongqiudi/special/:id', require('./routes/dongqiudi/special'));
|
||||
router.get('/dongqiudi/top_news', require('./routes/dongqiudi/top_news'));
|
||||
|
||||
// 维基百科 Wikipedia
|
||||
router.get('/wikipedia/mainland', require('./routes/wikipedia/mainland'));
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
const axios = require('../../utils/axios');
|
||||
const cheerio = require('cheerio');
|
||||
const utils = require('./utils');
|
||||
const date = require('../../utils/date');
|
||||
|
||||
module.exports = async (ctx) => {
|
||||
const id = ctx.params.id;
|
||||
const response = await axios.get(`https://www.dongqiudi.com/special/${id}`);
|
||||
|
||||
const $ = cheerio.load(response.data);
|
||||
|
||||
const host = 'https://www.dongqiudi.com';
|
||||
|
||||
const list = $('.detail.special ul li h3')
|
||||
.slice(0, 5)
|
||||
.get();
|
||||
|
||||
const proList = [];
|
||||
|
||||
const out = await Promise.all(
|
||||
list.map(async (item) => {
|
||||
const $ = cheerio.load(item);
|
||||
const title = $('a').text();
|
||||
const itemUrl = host + $('a').attr('href');
|
||||
|
||||
const cache = await ctx.cache.get(itemUrl);
|
||||
if (cache) {
|
||||
return Promise.resolve(JSON.parse(cache));
|
||||
}
|
||||
|
||||
const single = {
|
||||
title,
|
||||
link: itemUrl,
|
||||
};
|
||||
|
||||
const es = axios.get(itemUrl);
|
||||
proList.push(es);
|
||||
return Promise.resolve(single);
|
||||
})
|
||||
);
|
||||
|
||||
const responses = await axios.all(proList);
|
||||
for (let i = 0; i < proList.length; i++) {
|
||||
const $ = utils.ProcessVideo(cheerio.load(responses[i].data));
|
||||
const full = $('div.detail');
|
||||
|
||||
out[i].description = utils.ProcessHref(full.find('div:nth-of-type(1)')).html();
|
||||
out[i].author = full.find('span.name').text();
|
||||
out[i].pubDate = date(
|
||||
full
|
||||
.find('span.time')
|
||||
.text()
|
||||
.trim(),
|
||||
8
|
||||
);
|
||||
|
||||
ctx.cache.set(out[i].itemUrl, JSON.stringify(out[i]), 24 * 60 * 60);
|
||||
}
|
||||
ctx.state.data = {
|
||||
title: `懂球帝专题-${id}`,
|
||||
link: `https://www.dongqiudi.com/special/${id}`,
|
||||
item: out.filter((e) => e !== undefined),
|
||||
};
|
||||
};
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
const axios = require('../../utils/axios');
|
||||
const cheerio = require('cheerio');
|
||||
const utils = require('./utils');
|
||||
const date = require('../../utils/date');
|
||||
|
||||
module.exports = async (ctx) => {
|
||||
const response = await axios.get('https://api.dongqiudi.com/app/tabs/iphone/1.json?mark=gif&version=576');
|
||||
const data = response.data.articles;
|
||||
|
||||
const proList = [];
|
||||
|
||||
const out = await Promise.all(
|
||||
data.map(async (item) => {
|
||||
const title = item.title;
|
||||
const itemUrl = item.share;
|
||||
|
||||
const cache = await ctx.cache.get(itemUrl);
|
||||
if (cache) {
|
||||
return Promise.resolve(JSON.parse(cache));
|
||||
}
|
||||
|
||||
const single = {
|
||||
title,
|
||||
link: itemUrl,
|
||||
};
|
||||
|
||||
const es = axios.get(itemUrl);
|
||||
proList.push(es);
|
||||
return Promise.resolve(single);
|
||||
})
|
||||
);
|
||||
|
||||
const responses = await axios.all(proList);
|
||||
for (let i = 0; i < proList.length; i++) {
|
||||
const $ = utils.ProcessVideo(cheerio.load(responses[i].data));
|
||||
const full = $('div.detail');
|
||||
|
||||
out[i].description = utils.ProcessHref(full.find('div:nth-of-type(1)')).html();
|
||||
out[i].author = full.find('span.name').text();
|
||||
out[i].pubDate = date(
|
||||
full
|
||||
.find('span.time')
|
||||
.text()
|
||||
.trim(),
|
||||
8
|
||||
);
|
||||
}
|
||||
ctx.state.data = {
|
||||
title: '懂球帝头条新闻',
|
||||
link: 'http://dongqiudi.com/',
|
||||
item: out.filter((e) => e !== undefined),
|
||||
};
|
||||
};
|
||||
|
|
@ -10,13 +10,22 @@ module.exports = async (ctx) => {
|
|||
Referer: `https://w.qnmlgb.tech/authors/${id}/`,
|
||||
},
|
||||
});
|
||||
const data = response.data.result.articles[0];
|
||||
const rawData = response.data.result.articles;
|
||||
|
||||
const data = [];
|
||||
|
||||
for (let i = 0; i < rawData.length; i++) {
|
||||
const subArticles = rawData[i].sub_articles;
|
||||
for (let j = 0; j < subArticles.length; j++) {
|
||||
data.push(subArticles[j]);
|
||||
}
|
||||
}
|
||||
|
||||
ctx.state.data = {
|
||||
title: `${data.author.nickname}微信公众号`,
|
||||
title: `${rawData[0].author.nickname}微信公众号`,
|
||||
link: `https://w.qnmlgb.tech/authors/${id}/`,
|
||||
description: data.author.profile_desc,
|
||||
item: data.sub_articles.map((item) => ({
|
||||
description: rawData[0].author.profile_desc,
|
||||
item: data.map((item) => ({
|
||||
title: item.article.title,
|
||||
description: `${item.article.digest}<img referrerpolicy="no-referrer" src="${item.article.cover}">`,
|
||||
pubDate: new Date(item.article.datetime * 1000).toUTCString(),
|
||||
|
|
|
|||
Loading…
Reference in New Issue