feat(route/weibo): Fetch visitor Cookies before accessing some APIs (#20534)

* feat(route/weibo): Fetch visitor Cookies before accessing some APIs

Signed-off-by: Rongrong <i@rong.moe>

* fix(route/weibo/utils): getCookies: return too early on slow connection

Signed-off-by: Rongrong <i@rong.moe>

* fix(route/weibo/utils): getCookies: Redirection assertion is too loose

Signed-off-by: Rongrong <i@rong.moe>

---------

Signed-off-by: Rongrong <i@rong.moe>
This commit is contained in:
Rongrong
2025-11-22 06:04:14 +08:00
committed by GitHub
parent 8c7cd6832e
commit 6eb460c33b
9 changed files with 220 additions and 112 deletions
+3 -6
View File
@@ -72,9 +72,8 @@ async function handler(ctx) {
url: 'https://m.weibo.cn/api/config',
headers: {
Referer: `https://m.weibo.cn/`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
Cookie: config.weibo.cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data.data.uid;
@@ -91,9 +90,8 @@ async function handler(ctx) {
url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
headers: {
Referer: `https://m.weibo.cn/u/${uid}`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
Cookie: config.weibo.cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data;
@@ -113,9 +111,8 @@ async function handler(ctx) {
url: 'https://m.weibo.cn/feed/friends',
headers: {
Referer: `https://m.weibo.cn/`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
Cookie: config.weibo.cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data.data;
+1 -2
View File
@@ -67,9 +67,8 @@ async function handler(ctx) {
url: `https://m.weibo.cn/feed/group?gid=${gid}`,
headers: {
Referer: `https://m.weibo.cn/`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
Cookie: config.weibo.cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data.data;
+26 -19
View File
@@ -14,8 +14,14 @@ export const route: Route = {
example: '/weibo/keyword/RSSHub',
parameters: { keyword: '你想订阅的微博关键词', routeParams: '额外参数;请参阅上面的说明和表格' },
features: {
requireConfig: false,
requirePuppeteer: false,
requireConfig: [
{
name: 'WEIBO_COOKIES',
optional: true,
description: '',
},
],
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
@@ -29,23 +35,24 @@ export const route: Route = {
async function handler(ctx) {
const keyword = ctx.req.param('keyword');
const data = await cache.tryGet(
`weibo:keyword:${keyword}`,
async () => {
const _r = await got({
method: 'get',
url: `https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D61%26q%3D${encodeURIComponent(keyword)}%26t%3D0`,
headers: {
Referer: `https://m.weibo.cn/p/searchall?containerid=100103type%3D1%26q%3D${encodeURIComponent(keyword)}`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
},
});
return _r.data.data.cards;
},
config.cache.routeExpire,
false
const data = await weiboUtils.tryWithCookies((cookies) =>
cache.tryGet(
`weibo:keyword:${keyword}`,
async () => {
const _r = await got({
method: 'get',
url: `https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D61%26q%3D${encodeURIComponent(keyword)}%26t%3D0`,
headers: {
Referer: `https://m.weibo.cn/p/searchall?containerid=100103type%3D1%26q%3D${encodeURIComponent(keyword)}`,
Cookie: cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data.data.cards;
},
config.cache.routeExpire,
false
)
);
const routeParams = querystring.parse(ctx.req.param('routeParams'));
+4
View File
@@ -8,6 +8,10 @@ export const namespace: Namespace = {
一个已知的例子为:部分视频因未知原因仅限中国大陆境内访问 (CDN 域名为 \`locallimit.us.sinaimg.cn\` 而非 \`f.video.weibocdn.com\`)。若一条微博含有这种视频且 RSSHub 实例部署在境外,抓取到的微博可能不含视频。将 RSSHub 部署在境内有助于抓取这种视频,但阅读器也必须处于境内网络环境以加载视频。
:::
::: warning
大部分路由均需要 Cookies 才能获取。优先使用 \`WEIBO_COOKIES\`;未设置时尝试使用 Puppeteer 获取访客 Cookies。部分路由不支持访客访问,则必须设置 \`WEIBO_COOKIES\`,详见各个路由的文档。
:::
对于微博内容,在 \`routeParams\` 参数中以 query string 格式指定选项,可以控制输出的样式
| 键 | 含义 | 接受的值 | 默认值 |
+20 -13
View File
@@ -6,7 +6,7 @@ import { config } from '@/config';
import { art } from '@/utils/render';
import { load } from 'cheerio';
import path from 'node:path';
// import weiboUtils from '../utils';
import weiboUtils from '../utils';
// Default hide all picture
let wpic = 'false';
@@ -27,8 +27,14 @@ export const route: Route = {
},
},
features: {
requireConfig: false,
requirePuppeteer: false,
requireConfig: [
{
name: 'WEIBO_COOKIES',
optional: true,
description: '',
},
],
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
@@ -50,16 +56,17 @@ async function handler(ctx) {
fullpic = ctx.req.query('fullpic') ?? 'false';
const {
data: { data },
} = await got({
method: 'get',
url: 'https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot&title=%E5%BE%AE%E5%8D%9A%E7%83%AD%E6%90%9C&extparam=filter_type%3Drealtimehot%26mi_cid%3D100103%26pos%3D0_0%26c_type%3D30%26display_time%3D1540538388&luicode=10000011&lfid=231583',
headers: {
Referer: 'https://s.weibo.com/top/summary?cate=realtimehot',
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
},
});
} = await weiboUtils.tryWithCookies((cookies) =>
got({
method: 'get',
url: 'https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot&title=%E5%BE%AE%E5%8D%9A%E7%83%AD%E6%90%9C&extparam=filter_type%3Drealtimehot%26mi_cid%3D100103%26pos%3D0_0%26c_type%3D30%26display_time%3D1540538388&luicode=10000011&lfid=231583',
headers: {
Referer: 'https://s.weibo.com/top/summary?cate=realtimehot',
Cookie: cookies,
...weiboUtils.apiHeaders,
},
})
);
let resultItems = null;
if (ctx.req.param('fulltext') === 'fulltext') {
+29 -22
View File
@@ -11,8 +11,14 @@ export const route: Route = {
example: '/weibo/super_index/1008084989d223732bf6f02f75ea30efad58a9/sort_time',
parameters: { id: '超话ID', type: '类型:见下表', routeParams: '额外参数;请参阅上面的说明和表格' },
features: {
requireConfig: false,
requirePuppeteer: false,
requireConfig: [
{
name: 'WEIBO_COOKIES',
optional: true,
description: '',
},
],
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
@@ -45,26 +51,27 @@ async function handler(ctx) {
const id = ctx.req.param('id');
const type = ctx.req.param('type') ?? 'feed';
const containerData = (await cache.tryGet(
`weibo:super_index:container:${id}:${type}`,
async () => {
const _r = await got('https://m.weibo.cn/api/container/getIndex', {
searchParams: queryString.stringify({
containerid: `${id}_-_${type}`,
luicode: '10000011',
lfid: `${id}_-_main`,
}),
headers: {
Referer: `https://m.weibo.cn/p/index?containerid=${id}_-_soul&luicode=10000011&lfid=${id}_-_main`,
'MWeibo-Pwa': '1',
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
'X-Requested-With': 'XMLHttpRequest',
},
});
return _r.data.data;
},
config.cache.routeExpire,
false
const containerData = (await weiboUtils.tryWithCookies((cookies) =>
cache.tryGet(
`weibo:super_index:container:${id}:${type}`,
async () => {
const _r = await got('https://m.weibo.cn/api/container/getIndex', {
searchParams: queryString.stringify({
containerid: `${id}_-_${type}`,
luicode: '10000011',
lfid: `${id}_-_main`,
}),
headers: {
Referer: `https://m.weibo.cn/p/index?containerid=${id}_-_soul&luicode=10000011&lfid=${id}_-_main`,
Cookie: cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data.data;
},
config.cache.routeExpire,
false
)
)) as {
cards?: Card[];
pageInfo?: {
+3 -6
View File
@@ -74,9 +74,8 @@ async function handler(ctx) {
url: 'https://m.weibo.cn/api/config',
headers: {
Referer: 'https://m.weibo.cn/',
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
Cookie: config.weibo.cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data.data.uid;
@@ -93,9 +92,8 @@ async function handler(ctx) {
url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
headers: {
Referer: `https://m.weibo.cn/u/${uid}`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
Cookie: config.weibo.cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data;
@@ -119,9 +117,8 @@ async function handler(ctx) {
url: `https://m.weibo.cn/api/container/getIndex?containerid=${bookmarkContainerId}&openApp=0`,
headers: {
Referer: 'https://m.weibo.cn/',
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
Cookie: config.weibo.cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data.data.cards;
+38 -35
View File
@@ -22,7 +22,7 @@ export const route: Route = {
description: '',
},
],
requirePuppeteer: false,
requirePuppeteer: true,
antiCrawler: true,
supportBT: false,
supportPodcast: false,
@@ -69,23 +69,25 @@ async function handler(ctx) {
showBloggerIcons = fallback(undefined, queryToBoolean(routeParams.showBloggerIcons), false) ? '1' : '0';
}
}
const containerData = await cache.tryGet(
`weibo:user:index:${uid}`,
async () => {
const _r = await got({
method: 'get',
url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
headers: {
Referer: `https://m.weibo.cn/u/${uid}`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
Cookie: config.weibo.cookies,
},
});
return _r.data;
},
config.cache.routeExpire,
false
const containerData = await weiboUtils.tryWithCookies((cookies) =>
cache.tryGet(
`weibo:user:index:${uid}`,
async () => {
const _r = await got({
method: 'get',
url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
headers: {
Referer: `https://m.weibo.cn/u/${uid}`,
Cookie: cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data;
},
config.cache.routeExpire,
false
)
);
const name = containerData.data.userInfo.screen_name;
@@ -93,23 +95,24 @@ async function handler(ctx) {
const profileImageUrl = containerData.data.userInfo.profile_image_url;
const containerId = containerData.data.tabsInfo.tabs.find((item) => item.tab_type === 'weibo').containerid;
const cards = await cache.tryGet(
`weibo:user:cards:${uid}:${containerId}`,
async () => {
const _r = await got({
method: 'get',
url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}&containerid=${containerId}`,
headers: {
Referer: `https://m.weibo.cn/u/${uid}`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
Cookie: config.weibo.cookies,
},
});
return _r.data.data.cards;
},
config.cache.routeExpire,
false
const cards = await weiboUtils.tryWithCookies((cookies) =>
cache.tryGet(
`weibo:user:cards:${uid}:${containerId}`,
async () => {
const _r = await got({
method: 'get',
url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}&containerid=${containerId}`,
headers: {
Referer: `https://m.weibo.cn/u/${uid}`,
Cookie: cookies,
...weiboUtils.apiHeaders,
},
});
return _r.data.data.cards;
},
config.cache.routeExpire,
false
)
);
let resultItems = await Promise.all(
+96 -9
View File
@@ -1,10 +1,103 @@
import cache from '@/utils/cache';
import { config } from '@/config';
import { getPuppeteerPage } from '@/utils/puppeteer';
import { getCookies } from '@/utils/puppeteer-utils';
import logger from '@/utils/logger';
import querystring from 'node:querystring';
import got from '@/utils/got';
import { load } from 'cheerio';
import { fallback, queryToBoolean, queryToInteger } from '@/utils/readable-social';
class RenewWeiboCookiesError extends Error {
constructor(message: string) {
super(message);
this.name = 'RenewWeiboCookiesError';
}
}
const weiboUtils = {
apiHeaders: {
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
},
RenewWeiboCookiesError,
getCookies: async (renew = false) => {
if (config.weibo.cookies) {
if (renew) {
throw new Error('Cookies expired. Please update WEIBO_COOKIES');
}
return config.weibo.cookies;
}
const cacheKey = 'weibo:guest-cookies';
if (renew) {
cache.set(cacheKey, '', 1);
}
return await cache.tryGet(cacheKey, async () => {
const url = 'https://m.weibo.cn/';
if (renew) {
logger.warn(`Renewing visitor Cookies from ${url}`);
} else {
logger.info(`Fetching visitor Cookies from ${url}`);
}
let times = 0;
const { page, destory } = await getPuppeteerPage(url, {
onBeforeLoad: async (page) => {
const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']);
await page.setUserAgent(weiboUtils.apiHeaders['User-Agent']);
await page.setRequestInterception(true);
page.on('request', (request) => {
// 1st: initial request, 302 to visitor.passport.weibo.cn; 2nd: auth ok
if (!expectResourceTypes.has(request.resourceType()) || times >= 2) {
request.abort();
return;
}
if (request.url().startsWith(url)) {
times++;
}
request.continue();
});
},
// networkidle2 returns too early if the connection is slow
gotoConfig: { waitUntil: 'networkidle0' },
});
const cookies: string = await getCookies(page, 'weibo.cn');
await destory();
if (times < 2) {
throw new Error(`Unexpected redirection. Last URL: ${page.url()}`);
}
if (!cookies) {
throw new Error(`Unable to fetch visitor cookies. Please set WEIBO_COOKIES. Last URL: ${page.url()}`);
}
return cookies;
});
},
tryWithCookies: (() => {
let errors: number = 0;
return async (callback: (cookies: string) => Promise<any>) => {
try {
return await callback(await weiboUtils.getCookies(false));
} catch (error: any) {
if (error.message?.includes('WEIBO_COOKIES')) {
throw error;
}
if (errors > 10) {
logger.warn(`Too many errors while fetching data from weibo API, renewing Cookies: ${error.message}`);
logger.info('Please open an issue on GitHub if renewing Cookies fixes the error');
} else if ((error.name === 'HTTPError' || error.name === 'FetchError') && error.status === 432) {
// empty
} else if (error.name === 'RenewWeiboCookiesError') {
// empty
} else {
errors++;
throw error;
}
errors = 0;
return await callback(await weiboUtils.getCookies(true));
}
};
})(),
formatTitle: (html) =>
html
.replaceAll(/<span class=["']url-icon["']><img\s[^>]*?alt=["']?([^>]+?)["']?\s[^>]*?\/?><\/span>/g, '$1') // 表情转换
@@ -262,9 +355,7 @@ const weiboUtils = {
const itemResponse = await got.get(link, {
headers: {
Referer: `https://m.weibo.cn/u/${uid}`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
...weiboUtils.apiHeaders,
},
});
return itemResponse.data.data;
@@ -334,9 +425,7 @@ const weiboUtils = {
const _response = await got.get(link, {
headers: {
Referer: `https://card.weibo.com/article/m/show/id/${articleId}`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
...weiboUtils.apiHeaders,
},
});
return _response.data;
@@ -416,9 +505,7 @@ const weiboUtils = {
const _response = await got.get(link, {
headers: {
Referer: `https://m.weibo.cn/detail/${id}`,
'MWeibo-Pwa': 1,
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
...weiboUtils.apiHeaders,
},
});
return _response.data;