diff --git a/docs/new-media.md b/docs/new-media.md index a31a91025..7bd1338d8 100644 --- a/docs/new-media.md +++ b/docs/new-media.md @@ -3382,6 +3382,19 @@ column 为 third 时可选的 category: +### 公众号 (微阅读来源) + + + +::: warning 注意 + +由于使用了一些针对反爬的缓解措施,本路由响应较慢。默认只抓取前 5 条,可通过 `?limit=` 改变(不推荐,容易被反爬)。\ +该网站使用 IP 甄别访客,且应用严格的每日阅读量限额 (约 15 次),请自建并确保正确配置缓存;如使用内存缓存而非 Redis 缓存,请增大缓存容量。该限额足够订阅至少 3 个公众号 (假设公众号每日仅更新一次);首页 / 分类页更新相当频繁,不推荐订阅。 + +::: + + + ### 公众号 (wxnmh.com 来源) diff --git a/lib/v2/wechat/data258.js b/lib/v2/wechat/data258.js new file mode 100644 index 000000000..6e86ead61 --- /dev/null +++ b/lib/v2/wechat/data258.js @@ -0,0 +1,137 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const timezone = require('@/utils/timezone'); +const { finishArticleItem } = require('@/utils/wechat-mp'); +const { RequestInProgressError } = require('@/errors'); +const wait = require('@/utils/wait'); + +const parsePage = ($item, hyperlinkSelector, timeSelector) => { + const hyperlink = $item.find(hyperlinkSelector); + const title = hyperlink.text(); + const link = hyperlink.attr('href'); + const pubDate = timezone(parseDate($item.find(timeSelector).text(), 'YYYY-MM-DD HH:mm'), 8); + return { + title, + link, + pubDate, + }; +}; + +module.exports = async (ctx) => { + // !!! here we must use a lock to prevent other requests to break the anti-anti-crawler workarounds !!! + if ((await ctx.cache.get('data258:lock', false)) === '1') { + throw new RequestInProgressError('Another request is in progress, please try again later.'); + } + // !!! here no need to acquire the lock, because the MP/category page has no crawler detection !!! + + const id = ctx.params.id; + + const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 5; + + const rootUrl = 'https://mp.data258.com'; + const pageUrl = id ? `${rootUrl}/article/category/${id}` : rootUrl; + + const response = await got(pageUrl); + const $ = cheerio.load(response.data); + + const title = $('head title').text(); + // title = title.endsWith('-微阅读') ? title.slice(0, title.length - 4) : title; + const description = $('meta[name="description"]').attr('content'); + + const categoryPage = $('ul.fly-list'); + + let items; + if (categoryPage && categoryPage.length) { + // got a category page + items = $(categoryPage) + .find('li') + .map((_, item) => parsePage($(item), 'h2 a', '.fly-list-info span')) + .get(); + } else { + // got an MP page + items = $('ul.jie-row li') + .map((_, item) => parsePage($(item), 'a.jie-title', '.layui-hide-xs')) + .get(); + } + + items = items.slice(0, limit); // limit to avoid being anti-crawled + + // !!! double-check !!! + if ((await ctx.cache.get('data258:lock', false)) === '1') { + throw new RequestInProgressError('Another request is in progress, please try again later.'); + } else { + // !!! here we acquire the lock because the jump page has crawler detection !!! + await ctx.cache.set('data258:lock', '1', 60); + } + + // !!! here we must use a for-loop to ensure the concurrency is 1 !!! + // !!! please do note that if you try to increase the concurrency, your IP will be banned for a long time !!! + + let err; // !!! let RSSHub throw an anti-crawler prompt if the route is empty !!! + + /* eslint-disable no-await-in-loop */ + for (const item of items) { + // https://mp.data258.com/wx?id=${id}&t={token}, id is a permanent hex, token is a temporary base64 + const cacheId = item.link.match(/id=([\da-f]+)/)[1]; + item.link = item.link.startsWith('http') ? item.link : `${rootUrl}${item.link}`; + const realLink = await ctx.cache.tryGet(`data258:${cacheId}`, async () => { + try { + // !!! here we must sleep 1s to avoid being anti-crawled !!! + // !!! please do note that if the interval is less than 1s, your IP will be banned for a long time !!! + await wait(1000); + + const response = await got.get(item.link, { + headers: { + Referer: pageUrl, // essential + }, + }); + if (response.data.includes('今日浏览次数已达上限')) { + // !!! as long as cache hits, the link will not be crawled and consume the limit !!! + // !!! so that's not a big problem if the RSSHub instance is self-hosted !!! + err = new got.RequestError(response.data, {}, response.request); + return null; + } + const $ = cheerio.load(response.data); + const jmpJS = $('script') + .filter((_, e) => $(e).html().includes('location.href')) + .html(); + return jmpJS.match(/location\.href='([^']+)'/)[1]; + } catch (e) { + err = e; + return null; + } + }); + if (realLink) { + item.link = realLink; + } else { + break; // being anti-crawled, immediately cancel following operations + } + } + /* eslint-enable no-await-in-loop */ + + // !!! release the lock, let it expire immediately since no need to keep it in cache !!! + await ctx.cache.set('data258:lock', '0', 1); + + // jump links are valid only for a short period of time, drop those un-jumped items + // http://mp.weixin.qq.com/s + items = items.filter((item) => item.link.match(/^https?:\/\/mp\.weixin\.qq\.com\/s/)); + + if (items.length === 0 && err) { + // !!! if each request is anti-crawled, the filtered items array will be empty !!! + // !!! let RSSHub throw an anti-crawler prompt !!! + throw err; + } + + await Promise.all(items.map(async (item) => await finishArticleItem(ctx, item, !!categoryPage))); + + ctx.state.data = { + title, + link: pageUrl, + description, + item: items, + }; +}; + +// TODO: login? the valid time for cookies seems to be short, and abusing account will probably get banned... +// TODO: fetch full article for the official RSS feed? unless someone who is VIP contributes their RSS feed for test... diff --git a/lib/v2/wechat/maintainer.js b/lib/v2/wechat/maintainer.js index 0540a9dfc..6924eabb1 100644 --- a/lib/v2/wechat/maintainer.js +++ b/lib/v2/wechat/maintainer.js @@ -1,3 +1,4 @@ module.exports = { '/feeddd/:id': ['TonyRL', 'Rongronggg9'], + '/data258/:id?': ['Rongronggg9'], }; diff --git a/lib/v2/wechat/radar.js b/lib/v2/wechat/radar.js index ec100b868..03b68ec02 100644 --- a/lib/v2/wechat/radar.js +++ b/lib/v2/wechat/radar.js @@ -9,4 +9,15 @@ module.exports = { }, ], }, + 'data258.com': { + _name: '微信', + mp: [ + { + title: '公众号 (微阅读来源)', + docs: 'https://docs.rsshub.app/new-media.html#wei-xin', + source: ['/', '/article/category/:id'], + target: '/wechat/data258/:id?', + }, + ], + }, }; diff --git a/lib/v2/wechat/router.js b/lib/v2/wechat/router.js index 8d47a59ba..7af4ca0b6 100644 --- a/lib/v2/wechat/router.js +++ b/lib/v2/wechat/router.js @@ -1,3 +1,4 @@ module.exports = function (router) { router.get('/feeddd/:id', require('./feeddd')); + router.get('/data258/:id?', require('./data258')); }; diff --git a/lib/v2/wechat/templates/description.art b/lib/v2/wechat/templates/description.art deleted file mode 100644 index 7755ee9f6..000000000 --- a/lib/v2/wechat/templates/description.art +++ /dev/null @@ -1 +0,0 @@ -{{@ desc }} diff --git a/lib/v2/wechat/templates/image.art b/lib/v2/wechat/templates/image.art deleted file mode 100644 index b51527379..000000000 --- a/lib/v2/wechat/templates/image.art +++ /dev/null @@ -1 +0,0 @@ -