fix(route/twitter): dynamically resolve GraphQL query IDs and fix production CookieAgent (#21544)
* fix(route/twitter): dynamically resolve GraphQL query IDs from Twitter JS bundles Twitter rotates GraphQL query IDs every 2-4 weeks, causing 404 errors. Instead of relying solely on hardcoded IDs, fetch and cache the latest query IDs from Twitter's client-web main.js bundle at runtime. - Add gql-id-resolver module to extract queryId/operationName pairs - Cache resolved IDs using CACHE_CONTENT_EXPIRE (Redis supported) - Fall back to hardcoded IDs if dynamic resolution fails - Update init() to trigger resolution before first API call Signed-off-by: yuguorui <yuguorui@pku.edu.cn> * fix(route/twitter): use undici.fetch directly to preserve CookieAgent dispatcher Both ofetch and the global wrappedFetch (request-rewriter) drop the dispatcher option when constructing a new Request object, which prevents the CookieAgent from attaching cookies in production builds. Use undici.fetch directly for Twitter API requests to preserve the dispatcher. Signed-off-by: yuguorui <yuguorui@pku.edu.cn> --------- Signed-off-by: yuguorui <yuguorui@pku.edu.cn>
This commit is contained in:
Guorui Yu
GitHub
parent
8ac41e7a39
commit
d8eaee5889
|
|
@ -3,7 +3,7 @@ import InvalidParameterError from '@/errors/types/invalid-parameter';
|
|||
import cache from '@/utils/cache';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
|
||||
import { baseUrl, gqlFeatures, gqlMap } from './constants';
|
||||
import { baseUrl, gqlFeatures, gqlMap, initGqlMap } from './constants';
|
||||
import { gatherLegacyFromData, paginationTweets, twitterGot } from './utils';
|
||||
|
||||
const getUserData = (id) =>
|
||||
|
|
@ -216,5 +216,5 @@ export default {
|
|||
getList,
|
||||
getHomeTimeline,
|
||||
getHomeLatestTimeline,
|
||||
init: () => {},
|
||||
init: initGqlMap,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,19 +1,14 @@
|
|||
import { buildGqlMap, fallbackIds, resolveQueryIds } from './gql-id-resolver';
|
||||
|
||||
const baseUrl = 'https://x.com/i/api';
|
||||
|
||||
const graphQLEndpointsPlain = [
|
||||
'/graphql/E3opETHurmVJflFsUBVuUQ/UserTweets',
|
||||
'/graphql/Yka-W8dz7RaEuQNkroPkYw/UserByScreenName',
|
||||
'/graphql/HJFjzBgCs16TqxewQOeLNg/HomeTimeline',
|
||||
'/graphql/DiTkXJgLqBBxCs7zaYsbtA/HomeLatestTimeline',
|
||||
'/graphql/bt4TKuFz4T7Ckk-VvQVSow/UserTweetsAndReplies',
|
||||
'/graphql/dexO_2tohK86JDudXXG3Yw/UserMedia',
|
||||
'/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId',
|
||||
'/graphql/UN1i3zUiCWa-6r-Uaho4fw/SearchTimeline',
|
||||
'/graphql/Pa45JvqZuKcW1plybfgBlQ/ListLatestTweetsTimeline',
|
||||
'/graphql/QuBlQ6SxNAQCt6-kBiCXCQ/TweetDetail',
|
||||
];
|
||||
// Initial gqlMap from fallback IDs, updated dynamically via initGqlMap()
|
||||
let gqlMap: Record<string, string> = buildGqlMap(fallbackIds);
|
||||
|
||||
const gqlMap = Object.fromEntries(graphQLEndpointsPlain.map((endpoint) => [endpoint.split('/')[3].replace(/V2$|Query$|QueryV2$/, ''), endpoint]));
|
||||
const initGqlMap = async () => {
|
||||
const queryIds = await resolveQueryIds();
|
||||
gqlMap = buildGqlMap(queryIds);
|
||||
};
|
||||
|
||||
const thirdPartySupportedAPI = ['UserByScreenName', 'UserByRestId', 'UserTweets', 'UserTweetsAndReplies', 'ListLatestTweetsTimeline', 'SearchTimeline', 'UserMedia'];
|
||||
|
||||
|
|
@ -114,4 +109,4 @@ const timelineParams = {
|
|||
|
||||
const bearerToken = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
|
||||
|
||||
export { baseUrl, bearerToken, gqlFeatures, gqlMap, thirdPartySupportedAPI, timelineParams };
|
||||
export { baseUrl, bearerToken, gqlFeatures, gqlMap, initGqlMap, thirdPartySupportedAPI, timelineParams };
|
||||
|
|
|
|||
|
|
@ -0,0 +1,116 @@
|
|||
import { config } from '@/config';
|
||||
import cache from '@/utils/cache';
|
||||
import logger from '@/utils/logger';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
|
||||
const CACHE_KEY = 'twitter:gql-query-ids';
|
||||
|
||||
// Hardcoded fallback IDs (last known working values)
|
||||
export const fallbackIds: Record<string, string> = {
|
||||
UserTweets: 'E3opETHurmVJflFsUBVuUQ',
|
||||
UserByScreenName: 'Yka-W8dz7RaEuQNkroPkYw',
|
||||
HomeTimeline: 'xhYBF94fPSp8ey64FfYXiA',
|
||||
HomeLatestTimeline: '0vp2Au9doTKsbn2vIk48Dg',
|
||||
UserTweetsAndReplies: 'bt4TKuFz4T7Ckk-VvQVSow',
|
||||
UserMedia: 'dexO_2tohK86JDudXXG3Yw',
|
||||
UserByRestId: 'Qw77dDjp9xCpUY-AXwt-yQ',
|
||||
SearchTimeline: 'UN1i3zUiCWa-6r-Uaho4fw',
|
||||
ListLatestTweetsTimeline: 'Pa45JvqZuKcW1plybfgBlQ',
|
||||
TweetDetail: 'QuBlQ6SxNAQCt6-kBiCXCQ',
|
||||
};
|
||||
|
||||
const operationNames = Object.keys(fallbackIds);
|
||||
|
||||
async function fetchTwitterPage(): Promise<string> {
|
||||
const response = await ofetch('https://x.com', {
|
||||
parseResponse: (txt) => txt,
|
||||
});
|
||||
return response as unknown as string;
|
||||
}
|
||||
|
||||
function extractQueryIds(scriptContent: string): Record<string, string> {
|
||||
const ids: Record<string, string> = {};
|
||||
const matches = scriptContent.matchAll(/queryId:"([^"]+?)".+?operationName:"([^"]+?)"/g);
|
||||
for (const match of matches) {
|
||||
const [, queryId, operationName] = match;
|
||||
if (operationNames.includes(operationName)) {
|
||||
ids[operationName] = queryId;
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
async function fetchAndExtractIds(): Promise<Record<string, string>> {
|
||||
const html = await fetchTwitterPage();
|
||||
|
||||
// Extract main.hash.js URL — it contains all the GraphQL query IDs we need
|
||||
const mainMatch = html.match(/\/client-web\/main\.([a-z0-9]+)\./);
|
||||
if (!mainMatch) {
|
||||
logger.warn('twitter gql-id-resolver: main.js URL not found in Twitter page');
|
||||
return {};
|
||||
}
|
||||
|
||||
const mainUrl = `https://abs.twimg.com/responsive-web/client-web/main.${mainMatch[1]}.js`;
|
||||
logger.debug(`twitter gql-id-resolver: fetching ${mainUrl}`);
|
||||
|
||||
const content = await ofetch(mainUrl, {
|
||||
parseResponse: (txt) => txt,
|
||||
});
|
||||
return extractQueryIds(content as unknown as string);
|
||||
}
|
||||
|
||||
let resolvePromise: Promise<Record<string, string>> | null = null;
|
||||
|
||||
export async function resolveQueryIds(): Promise<Record<string, string>> {
|
||||
// Check cache first
|
||||
const cached = await cache.get(CACHE_KEY);
|
||||
if (cached) {
|
||||
try {
|
||||
const parsed = typeof cached === 'string' ? JSON.parse(cached) : cached;
|
||||
if (parsed && typeof parsed === 'object' && Object.keys(parsed).length > 0) {
|
||||
logger.debug(`twitter gql-id-resolver: using cached query IDs`);
|
||||
return { ...fallbackIds, ...parsed };
|
||||
}
|
||||
} catch {
|
||||
// ignore parse error
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate concurrent requests
|
||||
if (!resolvePromise) {
|
||||
resolvePromise = (async () => {
|
||||
try {
|
||||
logger.info('twitter gql-id-resolver: fetching fresh query IDs from Twitter JS bundles');
|
||||
const ids = await fetchAndExtractIds();
|
||||
|
||||
if (Object.keys(ids).length > 0) {
|
||||
await cache.set(CACHE_KEY, JSON.stringify(ids), config.cache.contentExpire);
|
||||
const found = operationNames.filter((name) => ids[name]);
|
||||
const missing = operationNames.filter((name) => !ids[name]);
|
||||
logger.debug(`twitter gql-id-resolver: resolved ${found.length}/${operationNames.length} query IDs. Missing: ${missing.join(', ') || 'none'}`);
|
||||
} else {
|
||||
logger.warn('twitter gql-id-resolver: failed to extract any query IDs, using fallback');
|
||||
}
|
||||
|
||||
return ids;
|
||||
} catch (error) {
|
||||
logger.warn(`twitter gql-id-resolver: error fetching query IDs: ${error}. Using fallback.`);
|
||||
return {};
|
||||
} finally {
|
||||
resolvePromise = null;
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
const ids = await resolvePromise;
|
||||
return { ...fallbackIds, ...ids };
|
||||
}
|
||||
|
||||
export function buildGqlMap(queryIds: Record<string, string>): Record<string, string> {
|
||||
const map: Record<string, string> = {};
|
||||
for (const name of operationNames) {
|
||||
const id = queryIds[name] || fallbackIds[name];
|
||||
map[name] = `/graphql/${id}/${name}`;
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import { cookie as HttpCookieAgentCookie, CookieAgent } from 'http-cookie-agent/undici';
|
||||
import queryString from 'query-string';
|
||||
import { Cookie, CookieJar } from 'tough-cookie';
|
||||
import { Client, ProxyAgent } from 'undici';
|
||||
import undici, { Client, ProxyAgent } from 'undici';
|
||||
|
||||
import { config } from '@/config';
|
||||
import ConfigNotFoundError from '@/errors/types/config-not-found';
|
||||
|
|
@ -136,8 +136,21 @@ export const twitterGot = async (
|
|||
)
|
||||
: {};
|
||||
|
||||
const response = await ofetch.raw(requestUrl, {
|
||||
retry: 0,
|
||||
// Use undici.fetch directly instead of ofetch.raw to preserve the CookieAgent
|
||||
// dispatcher. Two layers drop it in the normal path:
|
||||
// 1. ofetch does not forward `dispatcher` to its internal fetch() call
|
||||
// 2. wrappedFetch (request-rewriter) does `new Request(input, init)` which
|
||||
// discards non-standard options like `dispatcher`
|
||||
// Additionally, setting `cookie` header manually doesn't work either because
|
||||
// the Fetch spec treats `cookie` as a forbidden header name, so
|
||||
// `new Request()` silently strips it.
|
||||
// The only way to send cookies via CookieAgent is to call undici.fetch with
|
||||
// the dispatcher option directly.
|
||||
//
|
||||
// Because undici.fetch is the standard Fetch API and does not support ofetch's
|
||||
// `onResponse` callback, the rate-limit and auth error handling that was
|
||||
// previously in `onResponse` is now inlined below.
|
||||
const response = await undici.fetch(requestUrl, {
|
||||
headers: {
|
||||
authority: 'x.com',
|
||||
accept: '*/*',
|
||||
|
|
@ -160,67 +173,77 @@ export const twitterGot = async (
|
|||
}),
|
||||
},
|
||||
dispatcher: dispatchers?.agent,
|
||||
onResponse: async ({ response }) => {
|
||||
const remaining = response.headers.get('x-rate-limit-remaining');
|
||||
const remainingInt = Number.parseInt(remaining || '0');
|
||||
const reset = response.headers.get('x-rate-limit-reset');
|
||||
logger.debug(
|
||||
`twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}, auth: ${JSON.stringify(auth)}, status: ${response.status}, data: ${JSON.stringify(response._data?.data)}, cookie: ${JSON.stringify(dispatchers?.jar.serializeSync())}`
|
||||
);
|
||||
if (auth) {
|
||||
if (remaining && remainingInt < 2 && reset) {
|
||||
const resetTime = new Date(Number.parseInt(reset) * 1000);
|
||||
const delay = (resetTime.getTime() - Date.now()) / 1000;
|
||||
logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`);
|
||||
await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay) * 2);
|
||||
} else if (response.status === 429 || JSON.stringify(response._data?.data) === '{"user":{}}') {
|
||||
logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`);
|
||||
await cache.set(`${lockPrefix}${auth.token}`, '1', 2000);
|
||||
} else if (response.status === 403 || response.status === 401) {
|
||||
const newCookie = await login({
|
||||
username: auth.username,
|
||||
password: auth.password,
|
||||
authenticationSecret: auth.authenticationSecret,
|
||||
});
|
||||
if (newCookie) {
|
||||
logger.debug(`twitter debug: reset twitter cookie for token ${auth.token}, ${newCookie}`);
|
||||
await cache.set(`twitter:cookie:${auth.token}`, newCookie, config.cache.contentExpire);
|
||||
logger.debug(`twitter debug: unlock twitter cookie for token ${auth.token} with error1`);
|
||||
await cache.set(`${lockPrefix}${auth.token}`, '', 1);
|
||||
} else {
|
||||
const tokenIndex = config.twitter.authToken?.indexOf(auth.token);
|
||||
if (tokenIndex !== undefined && tokenIndex !== -1) {
|
||||
config.twitter.authToken?.splice(tokenIndex, 1);
|
||||
}
|
||||
if (auth.username) {
|
||||
const usernameIndex = config.twitter.username?.indexOf(auth.username);
|
||||
if (usernameIndex !== undefined && usernameIndex !== -1) {
|
||||
config.twitter.username?.splice(usernameIndex, 1);
|
||||
}
|
||||
}
|
||||
if (auth.password) {
|
||||
const passwordIndex = config.twitter.password?.indexOf(auth.password);
|
||||
if (passwordIndex !== undefined && passwordIndex !== -1) {
|
||||
config.twitter.password?.splice(passwordIndex, 1);
|
||||
}
|
||||
}
|
||||
logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`);
|
||||
await cache.set(`${lockPrefix}${auth.token}`, '1', 3600);
|
||||
}
|
||||
} else {
|
||||
logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`);
|
||||
await cache.set(`${lockPrefix}${auth.token}`, '', 1);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
let responseData: any;
|
||||
try {
|
||||
responseData = await response.json();
|
||||
} catch {
|
||||
responseData = null;
|
||||
}
|
||||
|
||||
// Handle rate limiting and auth errors
|
||||
const remaining = response.headers.get('x-rate-limit-remaining');
|
||||
const remainingInt = Number.parseInt(remaining || '0');
|
||||
const reset = response.headers.get('x-rate-limit-reset');
|
||||
logger.debug(
|
||||
`twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}, auth: ${JSON.stringify(auth)}, status: ${response.status}, data: ${JSON.stringify(responseData?.data)}, cookie: ${JSON.stringify(dispatchers?.jar.serializeSync())}`
|
||||
);
|
||||
if (auth) {
|
||||
if (remaining && remainingInt < 2 && reset) {
|
||||
const resetTime = new Date(Number.parseInt(reset) * 1000);
|
||||
const delay = (resetTime.getTime() - Date.now()) / 1000;
|
||||
logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`);
|
||||
await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay) * 2);
|
||||
} else if (response.status === 429 || JSON.stringify(responseData?.data) === '{"user":{}}') {
|
||||
logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`);
|
||||
await cache.set(`${lockPrefix}${auth.token}`, '1', 2000);
|
||||
} else if (response.status === 403 || response.status === 401) {
|
||||
const newCookie = await login({
|
||||
username: auth.username,
|
||||
password: auth.password,
|
||||
authenticationSecret: auth.authenticationSecret,
|
||||
});
|
||||
if (newCookie) {
|
||||
logger.debug(`twitter debug: reset twitter cookie for token ${auth.token}, ${newCookie}`);
|
||||
await cache.set(`twitter:cookie:${auth.token}`, newCookie, config.cache.contentExpire);
|
||||
await cache.set(`${lockPrefix}${auth.token}`, '', 1);
|
||||
} else {
|
||||
const tokenIndex = config.twitter.authToken?.indexOf(auth.token);
|
||||
if (tokenIndex !== undefined && tokenIndex !== -1) {
|
||||
config.twitter.authToken?.splice(tokenIndex, 1);
|
||||
}
|
||||
if (auth.username) {
|
||||
const usernameIndex = config.twitter.username?.indexOf(auth.username);
|
||||
if (usernameIndex !== undefined && usernameIndex !== -1) {
|
||||
config.twitter.username?.splice(usernameIndex, 1);
|
||||
}
|
||||
}
|
||||
if (auth.password) {
|
||||
const passwordIndex = config.twitter.password?.indexOf(auth.password);
|
||||
if (passwordIndex !== undefined && passwordIndex !== -1) {
|
||||
config.twitter.password?.splice(passwordIndex, 1);
|
||||
}
|
||||
}
|
||||
logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`);
|
||||
await cache.set(`${lockPrefix}${auth.token}`, '1', 3600);
|
||||
}
|
||||
} else {
|
||||
logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`);
|
||||
await cache.set(`${lockPrefix}${auth.token}`, '', 1);
|
||||
}
|
||||
}
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new Error(`Twitter API error: ${response.status}`);
|
||||
}
|
||||
|
||||
if (auth?.token) {
|
||||
logger.debug(`twitter debug: update twitter cookie for token ${auth.token}`);
|
||||
await cache.set(`twitter:cookie:${auth.token}`, JSON.stringify(dispatchers?.jar.serializeSync()), config.cache.contentExpire);
|
||||
}
|
||||
|
||||
return response._data;
|
||||
return responseData;
|
||||
};
|
||||
|
||||
export const paginationTweets = async (endpoint: string, userId: number | undefined, variables: Record<string, any>, path?: string[]) => {
|
||||
|
|
|
|||
Loading…
Reference in New Issue