From f936581d398c1bc4fb60c130f3aa9de924cb6f8d Mon Sep 17 00:00:00 2001 From: pseudoyu Date: Mon, 3 Mar 2025 22:49:36 +0800 Subject: [PATCH] feat(route/threads): use http2 to resolve threads 429 issue --- lib/routes/threads/index.ts | 110 ++++++++++---------- lib/routes/threads/utils.ts | 199 ++++++++++++++++++++---------------- package.json | 1 + pnpm-lock.yaml | 40 ++++++++ 4 files changed, 211 insertions(+), 139 deletions(-) diff --git a/lib/routes/threads/index.ts b/lib/routes/threads/index.ts index 9d3af631c..4fe9b657d 100644 --- a/lib/routes/threads/index.ts +++ b/lib/routes/threads/index.ts @@ -1,10 +1,8 @@ import { Route, ViewType } from '@/types'; -import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -import { REPLIES_QUERY, THREADS_QUERY, apiUrl, threadUrl, profileUrl, extractTokens, makeHeader, getUserId, buildContent } from './utils'; -import { destr } from 'destr'; -import cache from '@/utils/cache'; -import { config } from '@/config'; +import { threadUrl, profileUrl, extractTokens, getUserId, buildContent, createClient, makeRequest } from './utils'; +import { JSDOM } from 'jsdom'; +import { JSONPath } from 'jsonpath-plus'; export const route: Route = { path: '/:user/:routeParams?', @@ -25,13 +23,7 @@ Specify options (in the format of query string) in parameter \`routeParams\` to | \`showAuthorAvatarInDesc\` | Show avatar of author in description (RSS body) (Not recommended if your RSS reader extracts images from description) | \`0\`/\`1\`/\`true\`/\`false\` | \`falseP\` | | \`showEmojiForQuotesAndReply\` | Use "🔁" instead of "QT", "↩ī¸" instead of "Re" | \`0\`/\`1\`/\`true\`/\`false\` | \`true\` | | \`showQuotedInTitle\` | Show quoted tweet in title | \`0\`/\`1\`/\`true\`/\`false\` | \`true\` | -| \`replies\` | Show replies | \`0\`/\`1\`/\`true\`/\`false\` | \`true\` | - -Specify different option values than default values to improve readability. The URL - -\`\`\` -https://rsshub.app/threads/zuck/showAuthorInTitle=1&showAuthorInDesc=1&showQuotedAuthorAvatarInDesc=1&showAuthorAvatarInDesc=1&showEmojiForQuotesAndReply=1&showQuotedInTitle=1 -\`\`\``, +| \`replies\` | Show replies | \`0\`/\`1\`/\`true\`/\`false\` | \`true\` |`, }, }, name: 'User timeline', @@ -42,10 +34,10 @@ https://rsshub.app/threads/zuck/showAuthorInTitle=1&showAuthorInDesc=1&showQuote async function handler(ctx) { const { user, routeParams } = ctx.req.param(); const { lsd } = await extractTokens(user); - const userId = await getUserId(user, lsd); + const userId = await getUserId(user); const params = new URLSearchParams(routeParams); - const debugJson = { + const debugJson: any = { params: routeParams, lsd, }; @@ -60,48 +52,45 @@ async function handler(ctx) { replies: params.get('replies') ?? false, }; - const threadsResponse = await cache.tryGet( - `threads:${userId}:${options.replies}`, - () => - ofetch(apiUrl, { - method: 'POST', - headers: { - ...makeHeader(user, lsd), - 'content-type': 'application/x-www-form-urlencoded', - }, - body: new URLSearchParams({ - lsd, - variables: JSON.stringify({ userID: userId }), - doc_id: String(options.replies ? REPLIES_QUERY : THREADS_QUERY), - }).toString(), - parseResponse: (txt) => destr(txt), - }), - config.cache.routeExpire, - false - ); + const client = await createClient(); + const response = await makeRequest(client, user); + const dom = new JSDOM(response.body); + + let threadsData: ThreadItem[] | null = null; + for (const el of dom.window.document.querySelectorAll('script[data-sjs]')) { + try { + const data = JSONPath({ + path: '$..thread_items[0]', + json: JSON.parse(el.textContent || ''), + }); + + if (data?.length > 0) { + threadsData = data as ThreadItem[]; + break; + } + } catch { + // Skip invalid JSON + } + } + + if (!threadsData) { + throw new Error('Failed to fetch thread data'); + } debugJson.profileId = userId; - debugJson.response = { - response: threadsResponse, - }; + debugJson.response = { response: threadsData }; - const threads = threadsResponse?.data?.mediaData?.threads || []; - const userData = threadsResponse?.data?.mediaData?.threads?.[0]?.thread_items?.[0]?.post?.user || {}; + const userData: ThreadUser = threadsData[0]?.post?.user || { username: user, profile_pic_url: '' }; - const items = threads.flatMap((thread) => - thread.thread_items - .filter((item) => user === item.post.user?.username) - .map((item) => { - const { title, description } = buildContent(item, options); - return { - author: user, - title, - description, - pubDate: parseDate(item.post.taken_at, 'X'), - link: threadUrl(item.post.code), - }; - }) - ); + const items = threadsData + .filter((item) => user === item.post.user?.username) + .map((item) => ({ + author: user, + title: buildContent(item, options).title, + description: buildContent(item, options).description, + pubDate: parseDate(item.post.taken_at, 'X'), + link: threadUrl(item.post.code), + })); debugJson.items = items; ctx.set('json', debugJson); @@ -110,7 +99,22 @@ async function handler(ctx) { title: `${user} (@${user}) on Threads`, link: profileUrl(user), image: userData?.profile_pic_url, - // description: userData.biography, item: items, }; } + +interface ThreadUser { + username: string; + profile_pic_url: string; +} + +interface ThreadItem { + post: { + user?: ThreadUser; + taken_at: number; + code: string; + caption?: { + text: string; + }; + }; +} diff --git a/lib/routes/threads/utils.ts b/lib/routes/threads/utils.ts index dde71adaf..3f5657b22 100644 --- a/lib/routes/threads/utils.ts +++ b/lib/routes/threads/utils.ts @@ -1,97 +1,136 @@ -import ofetch from '@/utils/ofetch'; import { load } from 'cheerio'; import dayjs from 'dayjs'; import cache from '@/utils/cache'; -import { destr } from 'destr'; import NotFoundError from '@/errors/types/not-found'; +import { connect, type ClientHttp2Session } from 'node:http2'; +import * as zlib from 'zlib'; +import { JSDOM } from 'jsdom'; +import { JSONPath } from 'jsonpath-plus'; const profileUrl = (user: string) => `https://www.threads.net/@${user}`; const threadUrl = (code: string) => `https://www.threads.net/t/${code}`; -const instagramUrl = (user: string) => `https://i.instagram.com/api/v1/users/web_profile_info/?username=${user}`; -const apiUrl = 'https://www.threads.net/api/graphql'; -// const PROFILE_QUERY = 23_996_318_473_300_828; // no longer works -const THREADS_QUERY = 6_232_751_443_445_612; -const REPLIES_QUERY = 6_307_072_669_391_286; -const USER_AGENT = 'Barcelona 289.0.0.77.109 Android'; -const appId = '238260118697367'; -const asbdId = '129477'; +const USER_AGENT = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1'; + +interface ResponseData { + statusCode: number | undefined; + headers: any; + body: string; +} + +export function createClient(): Promise { + return new Promise((resolve, reject) => { + const client = connect('https://www.threads.net', {}); + + client.on('error', reject); + client.on('connect', () => resolve(client)); + client.on('timeout', () => reject(new Error('Connection timeout'))); + }); +} + +export const makeRequest = (client: ClientHttp2Session, user: string): Promise => + new Promise((resolve, reject) => { + const req = client.request({ + ':path': `/@${user}`, + Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Encoding': 'gzip, br', + 'Accept-Language': 'zh-CN,zh;q=0.9', + 'Cache-Control': 'no-cache', + Pragma: 'no-cache', + Priority: 'u=0, i', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Upgrade-Insecure-Requests': '1', + 'User-Agent': USER_AGENT, + }); + + req.on('response', (headers) => { + const contentEncoding = headers['content-encoding']; + let encodingStream = req; + + if (contentEncoding === 'gzip') { + // @ts-ignore + encodingStream = encodingStream.pipe(zlib.createGunzip()); + } else if (contentEncoding === 'br') { + // @ts-ignore + encodingStream = encodingStream.pipe(zlib.createBrotliDecompress()); + } + + encodingStream.setEncoding('utf8'); + + let data = ''; + encodingStream.on('data', (chunk) => { + data += chunk; + }); + + encodingStream.on('end', () => { + resolve({ + statusCode: 200, + headers: data, + body: data, + }); + }); + }); + + req.on('timeout', () => { + req.destroy(); + reject(new Error('Request timeout')); + }); + + req.on('error', reject); + req.end(); + }); const extractTokens = async (user): Promise<{ lsd: string }> => { - const response = await ofetch(profileUrl(user), { - headers: { - 'User-Agent': USER_AGENT, - 'X-IG-App-ID': appId, - }, - }); - const $ = load(response); + const client = await createClient(); + const response = await makeRequest(client, user); + const $ = load(response.body); const data = $('script:contains("LSD"):first').text(); - const lsd = data.match(/"LSD",\[],{"token":"([\w@-]+)"},/)?.[1]; + if (!lsd) { throw new NotFoundError('LSD token not found'); } - // const userId = data.match(/{"user_id":"(\d+)"},/)?.[1]; - - const ret = { lsd }; - return ret; + return { lsd }; }; -const makeHeader = (user: string, lsd: string) => ({ - Accept: '*/*', - Host: 'www.threads.net', - Origin: 'https://www.threads.net', - Referer: profileUrl(user), - 'User-Agent': USER_AGENT, - 'X-FB-LSD': lsd, - 'X-IG-App-ID': appId, - 'Sec-Fetch-Site': 'same-origin', -}); +const getUserId = (user: string): Promise => + cache + .tryGet(`threads:userId:${user}`, async () => { + const client = await createClient(); + const response = await makeRequest(client, user); + const dom = new JSDOM(response.body); -// the formal way always reachs the rate limit, so use instagram api to get user id instead -const getUserId = (user: string, lsd: string): Promise => - cache.tryGet(`threads:userId:${user}`, async () => { - const pathName = `/@${user}`; - const payload: any = { - 'route_urls[0]': pathName, - __a: '1', - __comet_req: '29', - lsd, - }; - const response = await ofetch('https://www.threads.net/ajax/bulk-route-definitions/', { - method: 'POST', - headers: { - ...makeHeader(user, lsd), - 'content-type': 'application/x-www-form-urlencoded', - 'X-ASBD-ID': asbdId, - }, - body: new URLSearchParams(payload).toString(), - parseResponse: (txt) => destr(txt.slice(9)), // remove "for (;;);" + for (const el of dom.window.document.querySelectorAll('script[data-sjs]')) { + try { + const data = JSONPath({ + path: '$..user_id', + json: JSON.parse(el.textContent || ''), + }); + + if (data?.[0]) { + return data[0]; + } + } catch { + // Skip invalid JSON + } + } + + throw new NotFoundError('User ID not found'); + }) + .then((result): string => { + if (!result || typeof result !== 'string') { + throw new TypeError('Invalid user ID type'); + } + return result; }); - let userId = response.payload.payloads[pathName].result.exports.rootView.props.user_id; - - if (!userId) { - const fallbackResponse = await ofetch(instagramUrl(user), { - headers: makeHeader(user, lsd), - }); - - if (!fallbackResponse?.data?.user) { - throw new NotFoundError('Instagram getUser API response is invalid'); - } - - userId = fallbackResponse.data.user.id; - if (!userId) { - throw new NotFoundError('User ID not found in Instagram getUser API response'); - } - } - - return userId; - }); - const hasMedia = (post) => post.image_versions2 || post.carousel_media || post.video_versions; + const buildMedia = (post) => { let html = ''; @@ -99,25 +138,13 @@ const buildMedia = (post) => { for (const media of post.carousel_media) { const firstImage = media.image_versions2?.candidates[0]; const firstVideo = media.video_versions?.[0]; - if (firstVideo) { - html += `'; - } else { - html += ``; - } + html += firstVideo ? `` : ``; } } else { const mainImage = post.image_versions2?.candidates?.[0]; const mainVideo = post.video_versions?.[0]; if (mainImage) { - if (mainVideo) { - html += `'; - } else { - html += ``; - } + html += mainVideo ? `` : ``; } } @@ -181,4 +208,4 @@ const buildContent = (item, options) => { return { title, description }; }; -export { apiUrl, profileUrl, threadUrl, THREADS_QUERY, REPLIES_QUERY, USER_AGENT, extractTokens, getUserId, makeHeader, hasMedia, buildMedia, buildContent }; +export { profileUrl, threadUrl, extractTokens, getUserId, buildContent }; diff --git a/package.json b/package.json index ab42de42b..cfa7e508c 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "ip-regex": "5.0.0", "jsdom": "26.0.0", "json-bigint": "1.0.0", + "jsonpath-plus": "^10.3.0", "jsrsasign": "10.9.0", "lru-cache": "11.0.2", "lz-string": "1.5.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 560ae4dec..1ed54a02d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -152,6 +152,9 @@ importers: json-bigint: specifier: 1.0.0 version: 1.0.0 + jsonpath-plus: + specifier: ^10.3.0 + version: 10.3.0 jsrsasign: specifier: 10.9.0 version: 10.9.0 @@ -1578,6 +1581,18 @@ packages: '@jridgewell/trace-mapping@0.3.25': resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} + '@jsep-plugin/assignment@1.3.0': + resolution: {integrity: sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ==} + engines: {node: '>= 10.16.0'} + peerDependencies: + jsep: ^0.4.0||^1.0.0 + + '@jsep-plugin/regex@1.0.4': + resolution: {integrity: sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg==} + engines: {node: '>= 10.16.0'} + peerDependencies: + jsep: ^0.4.0||^1.0.0 + '@lezer/common@1.2.3': resolution: {integrity: sha512-w7ojc8ejBqr2REPsWxJjrMFsA/ysDCFICn8zEOR9mrqzOu2amhITYuLD8ag6XZf0CFXDrhKqw7+tW8cX66NaDA==} @@ -4538,6 +4553,10 @@ packages: canvas: optional: true + jsep@1.4.0: + resolution: {integrity: sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==} + engines: {node: '>= 10.16.0'} + jsesc@3.0.2: resolution: {integrity: sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==} engines: {node: '>=6'} @@ -4584,6 +4603,11 @@ packages: jsonfile@6.1.0: resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==} + jsonpath-plus@10.3.0: + resolution: {integrity: sha512-8TNmfeTCk2Le33A3vRRwtuworG/L5RrgMvdjhKZxvyShO+mBu2fP50OWUjRLNtvw344DdDarFh9buFAZs5ujeA==} + engines: {node: '>=18.0.0'} + hasBin: true + jsonpointer@5.0.1: resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} engines: {node: '>=0.10.0'} @@ -8059,6 +8083,14 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.0 + '@jsep-plugin/assignment@1.3.0(jsep@1.4.0)': + dependencies: + jsep: 1.4.0 + + '@jsep-plugin/regex@1.0.4(jsep@1.4.0)': + dependencies: + jsep: 1.4.0 + '@lezer/common@1.2.3': {} '@lezer/css@1.1.10': @@ -11720,6 +11752,8 @@ snapshots: - supports-color - utf-8-validate + jsep@1.4.0: {} + jsesc@3.0.2: {} jsesc@3.1.0: {} @@ -11752,6 +11786,12 @@ snapshots: optionalDependencies: graceful-fs: 4.2.11 + jsonpath-plus@10.3.0: + dependencies: + '@jsep-plugin/assignment': 1.3.0(jsep@1.4.0) + '@jsep-plugin/regex': 1.0.4(jsep@1.4.0) + jsep: 1.4.0 + jsonpointer@5.0.1: {} jsprim@1.4.2: