diff --git a/lib/routes/weibo/friends.ts b/lib/routes/weibo/friends.ts index 6f3207ae3..c14ec94b8 100644 --- a/lib/routes/weibo/friends.ts +++ b/lib/routes/weibo/friends.ts @@ -72,9 +72,8 @@ async function handler(ctx) { url: 'https://m.weibo.cn/api/config', headers: { Referer: `https://m.weibo.cn/`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', Cookie: config.weibo.cookies, + ...weiboUtils.apiHeaders, }, }); return _r.data.data.uid; @@ -91,9 +90,8 @@ async function handler(ctx) { url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`, headers: { Referer: `https://m.weibo.cn/u/${uid}`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', Cookie: config.weibo.cookies, + ...weiboUtils.apiHeaders, }, }); return _r.data; @@ -113,9 +111,8 @@ async function handler(ctx) { url: 'https://m.weibo.cn/feed/friends', headers: { Referer: `https://m.weibo.cn/`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', Cookie: config.weibo.cookies, + ...weiboUtils.apiHeaders, }, }); return _r.data.data; diff --git a/lib/routes/weibo/group.ts b/lib/routes/weibo/group.ts index bc7963bf3..74e79e636 100644 --- a/lib/routes/weibo/group.ts +++ b/lib/routes/weibo/group.ts @@ -67,9 +67,8 @@ async function handler(ctx) { url: `https://m.weibo.cn/feed/group?gid=${gid}`, headers: { Referer: `https://m.weibo.cn/`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', Cookie: config.weibo.cookies, + ...weiboUtils.apiHeaders, }, }); return _r.data.data; diff --git a/lib/routes/weibo/keyword.ts b/lib/routes/weibo/keyword.ts index aaf1ed9d0..0084b515b 100644 --- a/lib/routes/weibo/keyword.ts +++ b/lib/routes/weibo/keyword.ts @@ -14,8 +14,14 @@ export const route: Route = { example: '/weibo/keyword/RSSHub', parameters: { keyword: '你想订阅的微博关键词', routeParams: '额外参数;请参阅上面的说明和表格' }, features: { - requireConfig: false, - requirePuppeteer: false, + requireConfig: [ + { + name: 'WEIBO_COOKIES', + optional: true, + description: '', + }, + ], + requirePuppeteer: true, antiCrawler: false, supportBT: false, supportPodcast: false, @@ -29,23 +35,24 @@ export const route: Route = { async function handler(ctx) { const keyword = ctx.req.param('keyword'); - const data = await cache.tryGet( - `weibo:keyword:${keyword}`, - async () => { - const _r = await got({ - method: 'get', - url: `https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D61%26q%3D${encodeURIComponent(keyword)}%26t%3D0`, - headers: { - Referer: `https://m.weibo.cn/p/searchall?containerid=100103type%3D1%26q%3D${encodeURIComponent(keyword)}`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', - }, - }); - return _r.data.data.cards; - }, - config.cache.routeExpire, - false + const data = await weiboUtils.tryWithCookies((cookies) => + cache.tryGet( + `weibo:keyword:${keyword}`, + async () => { + const _r = await got({ + method: 'get', + url: `https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D61%26q%3D${encodeURIComponent(keyword)}%26t%3D0`, + headers: { + Referer: `https://m.weibo.cn/p/searchall?containerid=100103type%3D1%26q%3D${encodeURIComponent(keyword)}`, + Cookie: cookies, + ...weiboUtils.apiHeaders, + }, + }); + return _r.data.data.cards; + }, + config.cache.routeExpire, + false + ) ); const routeParams = querystring.parse(ctx.req.param('routeParams')); diff --git a/lib/routes/weibo/namespace.ts b/lib/routes/weibo/namespace.ts index f1d07f387..99fdf0512 100644 --- a/lib/routes/weibo/namespace.ts +++ b/lib/routes/weibo/namespace.ts @@ -8,6 +8,10 @@ export const namespace: Namespace = { 一个已知的例子为:部分视频因未知原因仅限中国大陆境内访问 (CDN 域名为 \`locallimit.us.sinaimg.cn\` 而非 \`f.video.weibocdn.com\`)。若一条微博含有这种视频且 RSSHub 实例部署在境外,抓取到的微博可能不含视频。将 RSSHub 部署在境内有助于抓取这种视频,但阅读器也必须处于境内网络环境以加载视频。 ::: +::: warning +大部分路由均需要 Cookies 才能获取。优先使用 \`WEIBO_COOKIES\`;未设置时尝试使用 Puppeteer 获取访客 Cookies。部分路由不支持访客访问,则必须设置 \`WEIBO_COOKIES\`,详见各个路由的文档。 +::: + 对于微博内容,在 \`routeParams\` 参数中以 query string 格式指定选项,可以控制输出的样式 | 键 | 含义 | 接受的值 | 默认值 | diff --git a/lib/routes/weibo/search/hot.ts b/lib/routes/weibo/search/hot.ts index a9448233a..f43192db0 100644 --- a/lib/routes/weibo/search/hot.ts +++ b/lib/routes/weibo/search/hot.ts @@ -6,7 +6,7 @@ import { config } from '@/config'; import { art } from '@/utils/render'; import { load } from 'cheerio'; import path from 'node:path'; -// import weiboUtils from '../utils'; +import weiboUtils from '../utils'; // Default hide all picture let wpic = 'false'; @@ -27,8 +27,14 @@ export const route: Route = { }, }, features: { - requireConfig: false, - requirePuppeteer: false, + requireConfig: [ + { + name: 'WEIBO_COOKIES', + optional: true, + description: '', + }, + ], + requirePuppeteer: true, antiCrawler: false, supportBT: false, supportPodcast: false, @@ -50,16 +56,17 @@ async function handler(ctx) { fullpic = ctx.req.query('fullpic') ?? 'false'; const { data: { data }, - } = await got({ - method: 'get', - url: 'https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot&title=%E5%BE%AE%E5%8D%9A%E7%83%AD%E6%90%9C&extparam=filter_type%3Drealtimehot%26mi_cid%3D100103%26pos%3D0_0%26c_type%3D30%26display_time%3D1540538388&luicode=10000011&lfid=231583', - headers: { - Referer: 'https://s.weibo.com/top/summary?cate=realtimehot', - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', - }, - }); + } = await weiboUtils.tryWithCookies((cookies) => + got({ + method: 'get', + url: 'https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot&title=%E5%BE%AE%E5%8D%9A%E7%83%AD%E6%90%9C&extparam=filter_type%3Drealtimehot%26mi_cid%3D100103%26pos%3D0_0%26c_type%3D30%26display_time%3D1540538388&luicode=10000011&lfid=231583', + headers: { + Referer: 'https://s.weibo.com/top/summary?cate=realtimehot', + Cookie: cookies, + ...weiboUtils.apiHeaders, + }, + }) + ); let resultItems = null; if (ctx.req.param('fulltext') === 'fulltext') { diff --git a/lib/routes/weibo/super-index.ts b/lib/routes/weibo/super-index.ts index c414d9ce5..a8f373065 100644 --- a/lib/routes/weibo/super-index.ts +++ b/lib/routes/weibo/super-index.ts @@ -11,8 +11,14 @@ export const route: Route = { example: '/weibo/super_index/1008084989d223732bf6f02f75ea30efad58a9/sort_time', parameters: { id: '超话ID', type: '类型:见下表', routeParams: '额外参数;请参阅上面的说明和表格' }, features: { - requireConfig: false, - requirePuppeteer: false, + requireConfig: [ + { + name: 'WEIBO_COOKIES', + optional: true, + description: '', + }, + ], + requirePuppeteer: true, antiCrawler: false, supportBT: false, supportPodcast: false, @@ -45,26 +51,27 @@ async function handler(ctx) { const id = ctx.req.param('id'); const type = ctx.req.param('type') ?? 'feed'; - const containerData = (await cache.tryGet( - `weibo:super_index:container:${id}:${type}`, - async () => { - const _r = await got('https://m.weibo.cn/api/container/getIndex', { - searchParams: queryString.stringify({ - containerid: `${id}_-_${type}`, - luicode: '10000011', - lfid: `${id}_-_main`, - }), - headers: { - Referer: `https://m.weibo.cn/p/index?containerid=${id}_-_soul&luicode=10000011&lfid=${id}_-_main`, - 'MWeibo-Pwa': '1', - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', - 'X-Requested-With': 'XMLHttpRequest', - }, - }); - return _r.data.data; - }, - config.cache.routeExpire, - false + const containerData = (await weiboUtils.tryWithCookies((cookies) => + cache.tryGet( + `weibo:super_index:container:${id}:${type}`, + async () => { + const _r = await got('https://m.weibo.cn/api/container/getIndex', { + searchParams: queryString.stringify({ + containerid: `${id}_-_${type}`, + luicode: '10000011', + lfid: `${id}_-_main`, + }), + headers: { + Referer: `https://m.weibo.cn/p/index?containerid=${id}_-_soul&luicode=10000011&lfid=${id}_-_main`, + Cookie: cookies, + ...weiboUtils.apiHeaders, + }, + }); + return _r.data.data; + }, + config.cache.routeExpire, + false + ) )) as { cards?: Card[]; pageInfo?: { diff --git a/lib/routes/weibo/user-bookmarks.ts b/lib/routes/weibo/user-bookmarks.ts index 55ffccecf..099e3a710 100644 --- a/lib/routes/weibo/user-bookmarks.ts +++ b/lib/routes/weibo/user-bookmarks.ts @@ -74,9 +74,8 @@ async function handler(ctx) { url: 'https://m.weibo.cn/api/config', headers: { Referer: 'https://m.weibo.cn/', - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', Cookie: config.weibo.cookies, + ...weiboUtils.apiHeaders, }, }); return _r.data.data.uid; @@ -93,9 +92,8 @@ async function handler(ctx) { url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`, headers: { Referer: `https://m.weibo.cn/u/${uid}`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', Cookie: config.weibo.cookies, + ...weiboUtils.apiHeaders, }, }); return _r.data; @@ -119,9 +117,8 @@ async function handler(ctx) { url: `https://m.weibo.cn/api/container/getIndex?containerid=${bookmarkContainerId}&openApp=0`, headers: { Referer: 'https://m.weibo.cn/', - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', Cookie: config.weibo.cookies, + ...weiboUtils.apiHeaders, }, }); return _r.data.data.cards; diff --git a/lib/routes/weibo/user.ts b/lib/routes/weibo/user.ts index ca4988434..945c318f0 100644 --- a/lib/routes/weibo/user.ts +++ b/lib/routes/weibo/user.ts @@ -22,7 +22,7 @@ export const route: Route = { description: '', }, ], - requirePuppeteer: false, + requirePuppeteer: true, antiCrawler: true, supportBT: false, supportPodcast: false, @@ -69,23 +69,25 @@ async function handler(ctx) { showBloggerIcons = fallback(undefined, queryToBoolean(routeParams.showBloggerIcons), false) ? '1' : '0'; } } - const containerData = await cache.tryGet( - `weibo:user:index:${uid}`, - async () => { - const _r = await got({ - method: 'get', - url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`, - headers: { - Referer: `https://m.weibo.cn/u/${uid}`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', - Cookie: config.weibo.cookies, - }, - }); - return _r.data; - }, - config.cache.routeExpire, - false + + const containerData = await weiboUtils.tryWithCookies((cookies) => + cache.tryGet( + `weibo:user:index:${uid}`, + async () => { + const _r = await got({ + method: 'get', + url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`, + headers: { + Referer: `https://m.weibo.cn/u/${uid}`, + Cookie: cookies, + ...weiboUtils.apiHeaders, + }, + }); + return _r.data; + }, + config.cache.routeExpire, + false + ) ); const name = containerData.data.userInfo.screen_name; @@ -93,23 +95,24 @@ async function handler(ctx) { const profileImageUrl = containerData.data.userInfo.profile_image_url; const containerId = containerData.data.tabsInfo.tabs.find((item) => item.tab_type === 'weibo').containerid; - const cards = await cache.tryGet( - `weibo:user:cards:${uid}:${containerId}`, - async () => { - const _r = await got({ - method: 'get', - url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}&containerid=${containerId}`, - headers: { - Referer: `https://m.weibo.cn/u/${uid}`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', - Cookie: config.weibo.cookies, - }, - }); - return _r.data.data.cards; - }, - config.cache.routeExpire, - false + const cards = await weiboUtils.tryWithCookies((cookies) => + cache.tryGet( + `weibo:user:cards:${uid}:${containerId}`, + async () => { + const _r = await got({ + method: 'get', + url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}&containerid=${containerId}`, + headers: { + Referer: `https://m.weibo.cn/u/${uid}`, + Cookie: cookies, + ...weiboUtils.apiHeaders, + }, + }); + return _r.data.data.cards; + }, + config.cache.routeExpire, + false + ) ); let resultItems = await Promise.all( diff --git a/lib/routes/weibo/utils.ts b/lib/routes/weibo/utils.ts index 04f053de9..fd3c881ef 100644 --- a/lib/routes/weibo/utils.ts +++ b/lib/routes/weibo/utils.ts @@ -1,10 +1,103 @@ import cache from '@/utils/cache'; +import { config } from '@/config'; +import { getPuppeteerPage } from '@/utils/puppeteer'; +import { getCookies } from '@/utils/puppeteer-utils'; +import logger from '@/utils/logger'; import querystring from 'node:querystring'; import got from '@/utils/got'; import { load } from 'cheerio'; import { fallback, queryToBoolean, queryToInteger } from '@/utils/readable-social'; +class RenewWeiboCookiesError extends Error { + constructor(message: string) { + super(message); + this.name = 'RenewWeiboCookiesError'; + } +} + const weiboUtils = { + apiHeaders: { + 'MWeibo-Pwa': 1, + 'X-Requested-With': 'XMLHttpRequest', + 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', + }, + RenewWeiboCookiesError, + getCookies: async (renew = false) => { + if (config.weibo.cookies) { + if (renew) { + throw new Error('Cookies expired. Please update WEIBO_COOKIES'); + } + return config.weibo.cookies; + } + + const cacheKey = 'weibo:guest-cookies'; + if (renew) { + cache.set(cacheKey, '', 1); + } + return await cache.tryGet(cacheKey, async () => { + const url = 'https://m.weibo.cn/'; + if (renew) { + logger.warn(`Renewing visitor Cookies from ${url}`); + } else { + logger.info(`Fetching visitor Cookies from ${url}`); + } + let times = 0; + const { page, destory } = await getPuppeteerPage(url, { + onBeforeLoad: async (page) => { + const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']); + await page.setUserAgent(weiboUtils.apiHeaders['User-Agent']); + await page.setRequestInterception(true); + page.on('request', (request) => { + // 1st: initial request, 302 to visitor.passport.weibo.cn; 2nd: auth ok + if (!expectResourceTypes.has(request.resourceType()) || times >= 2) { + request.abort(); + return; + } + if (request.url().startsWith(url)) { + times++; + } + request.continue(); + }); + }, + // networkidle2 returns too early if the connection is slow + gotoConfig: { waitUntil: 'networkidle0' }, + }); + const cookies: string = await getCookies(page, 'weibo.cn'); + await destory(); + if (times < 2) { + throw new Error(`Unexpected redirection. Last URL: ${page.url()}`); + } + if (!cookies) { + throw new Error(`Unable to fetch visitor cookies. Please set WEIBO_COOKIES. Last URL: ${page.url()}`); + } + return cookies; + }); + }, + tryWithCookies: (() => { + let errors: number = 0; + return async (callback: (cookies: string) => Promise) => { + try { + return await callback(await weiboUtils.getCookies(false)); + } catch (error: any) { + if (error.message?.includes('WEIBO_COOKIES')) { + throw error; + } + if (errors > 10) { + logger.warn(`Too many errors while fetching data from weibo API, renewing Cookies: ${error.message}`); + logger.info('Please open an issue on GitHub if renewing Cookies fixes the error'); + } else if ((error.name === 'HTTPError' || error.name === 'FetchError') && error.status === 432) { + // empty + } else if (error.name === 'RenewWeiboCookiesError') { + // empty + } else { + errors++; + throw error; + } + errors = 0; + return await callback(await weiboUtils.getCookies(true)); + } + }; + })(), formatTitle: (html) => html .replaceAll(/]*?alt=["']?([^>]+?)["']?\s[^>]*?\/?><\/span>/g, '$1') // 表情转换 @@ -262,9 +355,7 @@ const weiboUtils = { const itemResponse = await got.get(link, { headers: { Referer: `https://m.weibo.cn/u/${uid}`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', + ...weiboUtils.apiHeaders, }, }); return itemResponse.data.data; @@ -334,9 +425,7 @@ const weiboUtils = { const _response = await got.get(link, { headers: { Referer: `https://card.weibo.com/article/m/show/id/${articleId}`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', + ...weiboUtils.apiHeaders, }, }); return _response.data; @@ -416,9 +505,7 @@ const weiboUtils = { const _response = await got.get(link, { headers: { Referer: `https://m.weibo.cn/detail/${id}`, - 'MWeibo-Pwa': 1, - 'X-Requested-With': 'XMLHttpRequest', - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', + ...weiboUtils.apiHeaders, }, }); return _response.data;