diff --git a/lib/routes/twitter/api/web-api/api.ts b/lib/routes/twitter/api/web-api/api.ts index be9c908cc..b412517ba 100644 --- a/lib/routes/twitter/api/web-api/api.ts +++ b/lib/routes/twitter/api/web-api/api.ts @@ -3,7 +3,7 @@ import InvalidParameterError from '@/errors/types/invalid-parameter'; import cache from '@/utils/cache'; import ofetch from '@/utils/ofetch'; -import { baseUrl, gqlFeatures, gqlMap } from './constants'; +import { baseUrl, gqlFeatures, gqlMap, initGqlMap } from './constants'; import { gatherLegacyFromData, paginationTweets, twitterGot } from './utils'; const getUserData = (id) => @@ -216,5 +216,5 @@ export default { getList, getHomeTimeline, getHomeLatestTimeline, - init: () => {}, + init: initGqlMap, }; diff --git a/lib/routes/twitter/api/web-api/constants.ts b/lib/routes/twitter/api/web-api/constants.ts index b1ca1d46a..ea58af273 100644 --- a/lib/routes/twitter/api/web-api/constants.ts +++ b/lib/routes/twitter/api/web-api/constants.ts @@ -1,19 +1,14 @@ +import { buildGqlMap, fallbackIds, resolveQueryIds } from './gql-id-resolver'; + const baseUrl = 'https://x.com/i/api'; -const graphQLEndpointsPlain = [ - '/graphql/E3opETHurmVJflFsUBVuUQ/UserTweets', - '/graphql/Yka-W8dz7RaEuQNkroPkYw/UserByScreenName', - '/graphql/HJFjzBgCs16TqxewQOeLNg/HomeTimeline', - '/graphql/DiTkXJgLqBBxCs7zaYsbtA/HomeLatestTimeline', - '/graphql/bt4TKuFz4T7Ckk-VvQVSow/UserTweetsAndReplies', - '/graphql/dexO_2tohK86JDudXXG3Yw/UserMedia', - '/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId', - '/graphql/UN1i3zUiCWa-6r-Uaho4fw/SearchTimeline', - '/graphql/Pa45JvqZuKcW1plybfgBlQ/ListLatestTweetsTimeline', - '/graphql/QuBlQ6SxNAQCt6-kBiCXCQ/TweetDetail', -]; +// Initial gqlMap from fallback IDs, updated dynamically via initGqlMap() +let gqlMap: Record = buildGqlMap(fallbackIds); -const gqlMap = Object.fromEntries(graphQLEndpointsPlain.map((endpoint) => [endpoint.split('/')[3].replace(/V2$|Query$|QueryV2$/, ''), endpoint])); +const initGqlMap = async () => { + const queryIds = await resolveQueryIds(); + gqlMap = buildGqlMap(queryIds); +}; const thirdPartySupportedAPI = ['UserByScreenName', 'UserByRestId', 'UserTweets', 'UserTweetsAndReplies', 'ListLatestTweetsTimeline', 'SearchTimeline', 'UserMedia']; @@ -114,4 +109,4 @@ const timelineParams = { const bearerToken = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; -export { baseUrl, bearerToken, gqlFeatures, gqlMap, thirdPartySupportedAPI, timelineParams }; +export { baseUrl, bearerToken, gqlFeatures, gqlMap, initGqlMap, thirdPartySupportedAPI, timelineParams }; diff --git a/lib/routes/twitter/api/web-api/gql-id-resolver.ts b/lib/routes/twitter/api/web-api/gql-id-resolver.ts new file mode 100644 index 000000000..9c2067ace --- /dev/null +++ b/lib/routes/twitter/api/web-api/gql-id-resolver.ts @@ -0,0 +1,116 @@ +import { config } from '@/config'; +import cache from '@/utils/cache'; +import logger from '@/utils/logger'; +import ofetch from '@/utils/ofetch'; + +const CACHE_KEY = 'twitter:gql-query-ids'; + +// Hardcoded fallback IDs (last known working values) +export const fallbackIds: Record = { + UserTweets: 'E3opETHurmVJflFsUBVuUQ', + UserByScreenName: 'Yka-W8dz7RaEuQNkroPkYw', + HomeTimeline: 'xhYBF94fPSp8ey64FfYXiA', + HomeLatestTimeline: '0vp2Au9doTKsbn2vIk48Dg', + UserTweetsAndReplies: 'bt4TKuFz4T7Ckk-VvQVSow', + UserMedia: 'dexO_2tohK86JDudXXG3Yw', + UserByRestId: 'Qw77dDjp9xCpUY-AXwt-yQ', + SearchTimeline: 'UN1i3zUiCWa-6r-Uaho4fw', + ListLatestTweetsTimeline: 'Pa45JvqZuKcW1plybfgBlQ', + TweetDetail: 'QuBlQ6SxNAQCt6-kBiCXCQ', +}; + +const operationNames = Object.keys(fallbackIds); + +async function fetchTwitterPage(): Promise { + const response = await ofetch('https://x.com', { + parseResponse: (txt) => txt, + }); + return response as unknown as string; +} + +function extractQueryIds(scriptContent: string): Record { + const ids: Record = {}; + const matches = scriptContent.matchAll(/queryId:"([^"]+?)".+?operationName:"([^"]+?)"/g); + for (const match of matches) { + const [, queryId, operationName] = match; + if (operationNames.includes(operationName)) { + ids[operationName] = queryId; + } + } + return ids; +} + +async function fetchAndExtractIds(): Promise> { + const html = await fetchTwitterPage(); + + // Extract main.hash.js URL — it contains all the GraphQL query IDs we need + const mainMatch = html.match(/\/client-web\/main\.([a-z0-9]+)\./); + if (!mainMatch) { + logger.warn('twitter gql-id-resolver: main.js URL not found in Twitter page'); + return {}; + } + + const mainUrl = `https://abs.twimg.com/responsive-web/client-web/main.${mainMatch[1]}.js`; + logger.debug(`twitter gql-id-resolver: fetching ${mainUrl}`); + + const content = await ofetch(mainUrl, { + parseResponse: (txt) => txt, + }); + return extractQueryIds(content as unknown as string); +} + +let resolvePromise: Promise> | null = null; + +export async function resolveQueryIds(): Promise> { + // Check cache first + const cached = await cache.get(CACHE_KEY); + if (cached) { + try { + const parsed = typeof cached === 'string' ? JSON.parse(cached) : cached; + if (parsed && typeof parsed === 'object' && Object.keys(parsed).length > 0) { + logger.debug(`twitter gql-id-resolver: using cached query IDs`); + return { ...fallbackIds, ...parsed }; + } + } catch { + // ignore parse error + } + } + + // Deduplicate concurrent requests + if (!resolvePromise) { + resolvePromise = (async () => { + try { + logger.info('twitter gql-id-resolver: fetching fresh query IDs from Twitter JS bundles'); + const ids = await fetchAndExtractIds(); + + if (Object.keys(ids).length > 0) { + await cache.set(CACHE_KEY, JSON.stringify(ids), config.cache.contentExpire); + const found = operationNames.filter((name) => ids[name]); + const missing = operationNames.filter((name) => !ids[name]); + logger.debug(`twitter gql-id-resolver: resolved ${found.length}/${operationNames.length} query IDs. Missing: ${missing.join(', ') || 'none'}`); + } else { + logger.warn('twitter gql-id-resolver: failed to extract any query IDs, using fallback'); + } + + return ids; + } catch (error) { + logger.warn(`twitter gql-id-resolver: error fetching query IDs: ${error}. Using fallback.`); + return {}; + } finally { + resolvePromise = null; + } + })(); + } + + const ids = await resolvePromise; + return { ...fallbackIds, ...ids }; +} + +export function buildGqlMap(queryIds: Record): Record { + const map: Record = {}; + for (const name of operationNames) { + const id = queryIds[name] || fallbackIds[name]; + map[name] = `/graphql/${id}/${name}`; + } + return map; +} diff --git a/lib/routes/twitter/api/web-api/utils.ts b/lib/routes/twitter/api/web-api/utils.ts index becf3f74a..560211aab 100644 --- a/lib/routes/twitter/api/web-api/utils.ts +++ b/lib/routes/twitter/api/web-api/utils.ts @@ -1,7 +1,7 @@ import { cookie as HttpCookieAgentCookie, CookieAgent } from 'http-cookie-agent/undici'; import queryString from 'query-string'; import { Cookie, CookieJar } from 'tough-cookie'; -import { Client, ProxyAgent } from 'undici'; +import undici, { Client, ProxyAgent } from 'undici'; import { config } from '@/config'; import ConfigNotFoundError from '@/errors/types/config-not-found'; @@ -136,8 +136,21 @@ export const twitterGot = async ( ) : {}; - const response = await ofetch.raw(requestUrl, { - retry: 0, + // Use undici.fetch directly instead of ofetch.raw to preserve the CookieAgent + // dispatcher. Two layers drop it in the normal path: + // 1. ofetch does not forward `dispatcher` to its internal fetch() call + // 2. wrappedFetch (request-rewriter) does `new Request(input, init)` which + // discards non-standard options like `dispatcher` + // Additionally, setting `cookie` header manually doesn't work either because + // the Fetch spec treats `cookie` as a forbidden header name, so + // `new Request()` silently strips it. + // The only way to send cookies via CookieAgent is to call undici.fetch with + // the dispatcher option directly. + // + // Because undici.fetch is the standard Fetch API and does not support ofetch's + // `onResponse` callback, the rate-limit and auth error handling that was + // previously in `onResponse` is now inlined below. + const response = await undici.fetch(requestUrl, { headers: { authority: 'x.com', accept: '*/*', @@ -160,67 +173,77 @@ export const twitterGot = async ( }), }, dispatcher: dispatchers?.agent, - onResponse: async ({ response }) => { - const remaining = response.headers.get('x-rate-limit-remaining'); - const remainingInt = Number.parseInt(remaining || '0'); - const reset = response.headers.get('x-rate-limit-reset'); - logger.debug( - `twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}, auth: ${JSON.stringify(auth)}, status: ${response.status}, data: ${JSON.stringify(response._data?.data)}, cookie: ${JSON.stringify(dispatchers?.jar.serializeSync())}` - ); - if (auth) { - if (remaining && remainingInt < 2 && reset) { - const resetTime = new Date(Number.parseInt(reset) * 1000); - const delay = (resetTime.getTime() - Date.now()) / 1000; - logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`); - await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay) * 2); - } else if (response.status === 429 || JSON.stringify(response._data?.data) === '{"user":{}}') { - logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`); - await cache.set(`${lockPrefix}${auth.token}`, '1', 2000); - } else if (response.status === 403 || response.status === 401) { - const newCookie = await login({ - username: auth.username, - password: auth.password, - authenticationSecret: auth.authenticationSecret, - }); - if (newCookie) { - logger.debug(`twitter debug: reset twitter cookie for token ${auth.token}, ${newCookie}`); - await cache.set(`twitter:cookie:${auth.token}`, newCookie, config.cache.contentExpire); - logger.debug(`twitter debug: unlock twitter cookie for token ${auth.token} with error1`); - await cache.set(`${lockPrefix}${auth.token}`, '', 1); - } else { - const tokenIndex = config.twitter.authToken?.indexOf(auth.token); - if (tokenIndex !== undefined && tokenIndex !== -1) { - config.twitter.authToken?.splice(tokenIndex, 1); - } - if (auth.username) { - const usernameIndex = config.twitter.username?.indexOf(auth.username); - if (usernameIndex !== undefined && usernameIndex !== -1) { - config.twitter.username?.splice(usernameIndex, 1); - } - } - if (auth.password) { - const passwordIndex = config.twitter.password?.indexOf(auth.password); - if (passwordIndex !== undefined && passwordIndex !== -1) { - config.twitter.password?.splice(passwordIndex, 1); - } - } - logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`); - await cache.set(`${lockPrefix}${auth.token}`, '1', 3600); - } - } else { - logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`); - await cache.set(`${lockPrefix}${auth.token}`, '', 1); - } - } - }, }); + let responseData: any; + try { + responseData = await response.json(); + } catch { + responseData = null; + } + + // Handle rate limiting and auth errors + const remaining = response.headers.get('x-rate-limit-remaining'); + const remainingInt = Number.parseInt(remaining || '0'); + const reset = response.headers.get('x-rate-limit-reset'); + logger.debug( + `twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}, auth: ${JSON.stringify(auth)}, status: ${response.status}, data: ${JSON.stringify(responseData?.data)}, cookie: ${JSON.stringify(dispatchers?.jar.serializeSync())}` + ); + if (auth) { + if (remaining && remainingInt < 2 && reset) { + const resetTime = new Date(Number.parseInt(reset) * 1000); + const delay = (resetTime.getTime() - Date.now()) / 1000; + logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`); + await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay) * 2); + } else if (response.status === 429 || JSON.stringify(responseData?.data) === '{"user":{}}') { + logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`); + await cache.set(`${lockPrefix}${auth.token}`, '1', 2000); + } else if (response.status === 403 || response.status === 401) { + const newCookie = await login({ + username: auth.username, + password: auth.password, + authenticationSecret: auth.authenticationSecret, + }); + if (newCookie) { + logger.debug(`twitter debug: reset twitter cookie for token ${auth.token}, ${newCookie}`); + await cache.set(`twitter:cookie:${auth.token}`, newCookie, config.cache.contentExpire); + await cache.set(`${lockPrefix}${auth.token}`, '', 1); + } else { + const tokenIndex = config.twitter.authToken?.indexOf(auth.token); + if (tokenIndex !== undefined && tokenIndex !== -1) { + config.twitter.authToken?.splice(tokenIndex, 1); + } + if (auth.username) { + const usernameIndex = config.twitter.username?.indexOf(auth.username); + if (usernameIndex !== undefined && usernameIndex !== -1) { + config.twitter.username?.splice(usernameIndex, 1); + } + } + if (auth.password) { + const passwordIndex = config.twitter.password?.indexOf(auth.password); + if (passwordIndex !== undefined && passwordIndex !== -1) { + config.twitter.password?.splice(passwordIndex, 1); + } + } + logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`); + await cache.set(`${lockPrefix}${auth.token}`, '1', 3600); + } + } else { + logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`); + await cache.set(`${lockPrefix}${auth.token}`, '', 1); + } + } + + if (response.status >= 400) { + throw new Error(`Twitter API error: ${response.status}`); + } + if (auth?.token) { logger.debug(`twitter debug: update twitter cookie for token ${auth.token}`); await cache.set(`twitter:cookie:${auth.token}`, JSON.stringify(dispatchers?.jar.serializeSync()), config.cache.contentExpire); } - return response._data; + return responseData; }; export const paginationTweets = async (endpoint: string, userId: number | undefined, variables: Record, path?: string[]) => {