feat(route/threads): use http2 to resolve threads 429 issue
This commit is contained in:
pseudoyu
parent
7e94e4e306
commit
f936581d39
|
|
@ -1,10 +1,8 @@
|
|||
import { Route, ViewType } from '@/types';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { REPLIES_QUERY, THREADS_QUERY, apiUrl, threadUrl, profileUrl, extractTokens, makeHeader, getUserId, buildContent } from './utils';
|
||||
import { destr } from 'destr';
|
||||
import cache from '@/utils/cache';
|
||||
import { config } from '@/config';
|
||||
import { threadUrl, profileUrl, extractTokens, getUserId, buildContent, createClient, makeRequest } from './utils';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { JSONPath } from 'jsonpath-plus';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/:user/:routeParams?',
|
||||
|
|
@ -25,13 +23,7 @@ Specify options (in the format of query string) in parameter \`routeParams\` to
|
|||
| \`showAuthorAvatarInDesc\` | Show avatar of author in description (RSS body) (Not recommended if your RSS reader extracts images from description) | \`0\`/\`1\`/\`true\`/\`false\` | \`falseP\` |
|
||||
| \`showEmojiForQuotesAndReply\` | Use "🔁" instead of "QT", "↩️" instead of "Re" | \`0\`/\`1\`/\`true\`/\`false\` | \`true\` |
|
||||
| \`showQuotedInTitle\` | Show quoted tweet in title | \`0\`/\`1\`/\`true\`/\`false\` | \`true\` |
|
||||
| \`replies\` | Show replies | \`0\`/\`1\`/\`true\`/\`false\` | \`true\` |
|
||||
|
||||
Specify different option values than default values to improve readability. The URL
|
||||
|
||||
\`\`\`
|
||||
https://rsshub.app/threads/zuck/showAuthorInTitle=1&showAuthorInDesc=1&showQuotedAuthorAvatarInDesc=1&showAuthorAvatarInDesc=1&showEmojiForQuotesAndReply=1&showQuotedInTitle=1
|
||||
\`\`\``,
|
||||
| \`replies\` | Show replies | \`0\`/\`1\`/\`true\`/\`false\` | \`true\` |`,
|
||||
},
|
||||
},
|
||||
name: 'User timeline',
|
||||
|
|
@ -42,10 +34,10 @@ https://rsshub.app/threads/zuck/showAuthorInTitle=1&showAuthorInDesc=1&showQuote
|
|||
async function handler(ctx) {
|
||||
const { user, routeParams } = ctx.req.param();
|
||||
const { lsd } = await extractTokens(user);
|
||||
const userId = await getUserId(user, lsd);
|
||||
const userId = await getUserId(user);
|
||||
|
||||
const params = new URLSearchParams(routeParams);
|
||||
const debugJson = {
|
||||
const debugJson: any = {
|
||||
params: routeParams,
|
||||
lsd,
|
||||
};
|
||||
|
|
@ -60,48 +52,45 @@ async function handler(ctx) {
|
|||
replies: params.get('replies') ?? false,
|
||||
};
|
||||
|
||||
const threadsResponse = await cache.tryGet(
|
||||
`threads:${userId}:${options.replies}`,
|
||||
() =>
|
||||
ofetch(apiUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
...makeHeader(user, lsd),
|
||||
'content-type': 'application/x-www-form-urlencoded',
|
||||
},
|
||||
body: new URLSearchParams({
|
||||
lsd,
|
||||
variables: JSON.stringify({ userID: userId }),
|
||||
doc_id: String(options.replies ? REPLIES_QUERY : THREADS_QUERY),
|
||||
}).toString(),
|
||||
parseResponse: (txt) => destr(txt),
|
||||
}),
|
||||
config.cache.routeExpire,
|
||||
false
|
||||
);
|
||||
const client = await createClient();
|
||||
const response = await makeRequest(client, user);
|
||||
const dom = new JSDOM(response.body);
|
||||
|
||||
let threadsData: ThreadItem[] | null = null;
|
||||
for (const el of dom.window.document.querySelectorAll('script[data-sjs]')) {
|
||||
try {
|
||||
const data = JSONPath({
|
||||
path: '$..thread_items[0]',
|
||||
json: JSON.parse(el.textContent || ''),
|
||||
});
|
||||
|
||||
if (data?.length > 0) {
|
||||
threadsData = data as ThreadItem[];
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// Skip invalid JSON
|
||||
}
|
||||
}
|
||||
|
||||
if (!threadsData) {
|
||||
throw new Error('Failed to fetch thread data');
|
||||
}
|
||||
|
||||
debugJson.profileId = userId;
|
||||
debugJson.response = {
|
||||
response: threadsResponse,
|
||||
};
|
||||
debugJson.response = { response: threadsData };
|
||||
|
||||
const threads = threadsResponse?.data?.mediaData?.threads || [];
|
||||
const userData = threadsResponse?.data?.mediaData?.threads?.[0]?.thread_items?.[0]?.post?.user || {};
|
||||
const userData: ThreadUser = threadsData[0]?.post?.user || { username: user, profile_pic_url: '' };
|
||||
|
||||
const items = threads.flatMap((thread) =>
|
||||
thread.thread_items
|
||||
.filter((item) => user === item.post.user?.username)
|
||||
.map((item) => {
|
||||
const { title, description } = buildContent(item, options);
|
||||
return {
|
||||
author: user,
|
||||
title,
|
||||
description,
|
||||
pubDate: parseDate(item.post.taken_at, 'X'),
|
||||
link: threadUrl(item.post.code),
|
||||
};
|
||||
})
|
||||
);
|
||||
const items = threadsData
|
||||
.filter((item) => user === item.post.user?.username)
|
||||
.map((item) => ({
|
||||
author: user,
|
||||
title: buildContent(item, options).title,
|
||||
description: buildContent(item, options).description,
|
||||
pubDate: parseDate(item.post.taken_at, 'X'),
|
||||
link: threadUrl(item.post.code),
|
||||
}));
|
||||
|
||||
debugJson.items = items;
|
||||
ctx.set('json', debugJson);
|
||||
|
|
@ -110,7 +99,22 @@ async function handler(ctx) {
|
|||
title: `${user} (@${user}) on Threads`,
|
||||
link: profileUrl(user),
|
||||
image: userData?.profile_pic_url,
|
||||
// description: userData.biography,
|
||||
item: items,
|
||||
};
|
||||
}
|
||||
|
||||
interface ThreadUser {
|
||||
username: string;
|
||||
profile_pic_url: string;
|
||||
}
|
||||
|
||||
interface ThreadItem {
|
||||
post: {
|
||||
user?: ThreadUser;
|
||||
taken_at: number;
|
||||
code: string;
|
||||
caption?: {
|
||||
text: string;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,97 +1,136 @@
|
|||
import ofetch from '@/utils/ofetch';
|
||||
import { load } from 'cheerio';
|
||||
import dayjs from 'dayjs';
|
||||
import cache from '@/utils/cache';
|
||||
import { destr } from 'destr';
|
||||
import NotFoundError from '@/errors/types/not-found';
|
||||
import { connect, type ClientHttp2Session } from 'node:http2';
|
||||
import * as zlib from 'zlib';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { JSONPath } from 'jsonpath-plus';
|
||||
|
||||
const profileUrl = (user: string) => `https://www.threads.net/@${user}`;
|
||||
const threadUrl = (code: string) => `https://www.threads.net/t/${code}`;
|
||||
const instagramUrl = (user: string) => `https://i.instagram.com/api/v1/users/web_profile_info/?username=${user}`;
|
||||
|
||||
const apiUrl = 'https://www.threads.net/api/graphql';
|
||||
// const PROFILE_QUERY = 23_996_318_473_300_828; // no longer works
|
||||
const THREADS_QUERY = 6_232_751_443_445_612;
|
||||
const REPLIES_QUERY = 6_307_072_669_391_286;
|
||||
const USER_AGENT = 'Barcelona 289.0.0.77.109 Android';
|
||||
const appId = '238260118697367';
|
||||
const asbdId = '129477';
|
||||
const USER_AGENT = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1';
|
||||
|
||||
interface ResponseData {
|
||||
statusCode: number | undefined;
|
||||
headers: any;
|
||||
body: string;
|
||||
}
|
||||
|
||||
export function createClient(): Promise<ClientHttp2Session> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const client = connect('https://www.threads.net', {});
|
||||
|
||||
client.on('error', reject);
|
||||
client.on('connect', () => resolve(client));
|
||||
client.on('timeout', () => reject(new Error('Connection timeout')));
|
||||
});
|
||||
}
|
||||
|
||||
export const makeRequest = (client: ClientHttp2Session, user: string): Promise<ResponseData> =>
|
||||
new Promise((resolve, reject) => {
|
||||
const req = client.request({
|
||||
':path': `/@${user}`,
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Encoding': 'gzip, br',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9',
|
||||
'Cache-Control': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
Priority: 'u=0, i',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'User-Agent': USER_AGENT,
|
||||
});
|
||||
|
||||
req.on('response', (headers) => {
|
||||
const contentEncoding = headers['content-encoding'];
|
||||
let encodingStream = req;
|
||||
|
||||
if (contentEncoding === 'gzip') {
|
||||
// @ts-ignore
|
||||
encodingStream = encodingStream.pipe(zlib.createGunzip());
|
||||
} else if (contentEncoding === 'br') {
|
||||
// @ts-ignore
|
||||
encodingStream = encodingStream.pipe(zlib.createBrotliDecompress());
|
||||
}
|
||||
|
||||
encodingStream.setEncoding('utf8');
|
||||
|
||||
let data = '';
|
||||
encodingStream.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
encodingStream.on('end', () => {
|
||||
resolve({
|
||||
statusCode: 200,
|
||||
headers: data,
|
||||
body: data,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
req.on('timeout', () => {
|
||||
req.destroy();
|
||||
reject(new Error('Request timeout'));
|
||||
});
|
||||
|
||||
req.on('error', reject);
|
||||
req.end();
|
||||
});
|
||||
|
||||
const extractTokens = async (user): Promise<{ lsd: string }> => {
|
||||
const response = await ofetch(profileUrl(user), {
|
||||
headers: {
|
||||
'User-Agent': USER_AGENT,
|
||||
'X-IG-App-ID': appId,
|
||||
},
|
||||
});
|
||||
const $ = load(response);
|
||||
const client = await createClient();
|
||||
const response = await makeRequest(client, user);
|
||||
const $ = load(response.body);
|
||||
|
||||
const data = $('script:contains("LSD"):first').text();
|
||||
|
||||
const lsd = data.match(/"LSD",\[],{"token":"([\w@-]+)"},/)?.[1];
|
||||
|
||||
if (!lsd) {
|
||||
throw new NotFoundError('LSD token not found');
|
||||
}
|
||||
|
||||
// const userId = data.match(/{"user_id":"(\d+)"},/)?.[1];
|
||||
|
||||
const ret = { lsd };
|
||||
return ret;
|
||||
return { lsd };
|
||||
};
|
||||
|
||||
const makeHeader = (user: string, lsd: string) => ({
|
||||
Accept: '*/*',
|
||||
Host: 'www.threads.net',
|
||||
Origin: 'https://www.threads.net',
|
||||
Referer: profileUrl(user),
|
||||
'User-Agent': USER_AGENT,
|
||||
'X-FB-LSD': lsd,
|
||||
'X-IG-App-ID': appId,
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
});
|
||||
const getUserId = (user: string): Promise<string> =>
|
||||
cache
|
||||
.tryGet(`threads:userId:${user}`, async () => {
|
||||
const client = await createClient();
|
||||
const response = await makeRequest(client, user);
|
||||
const dom = new JSDOM(response.body);
|
||||
|
||||
// the formal way always reachs the rate limit, so use instagram api to get user id instead
|
||||
const getUserId = (user: string, lsd: string): Promise<string> =>
|
||||
cache.tryGet(`threads:userId:${user}`, async () => {
|
||||
const pathName = `/@${user}`;
|
||||
const payload: any = {
|
||||
'route_urls[0]': pathName,
|
||||
__a: '1',
|
||||
__comet_req: '29',
|
||||
lsd,
|
||||
};
|
||||
const response = await ofetch('https://www.threads.net/ajax/bulk-route-definitions/', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
...makeHeader(user, lsd),
|
||||
'content-type': 'application/x-www-form-urlencoded',
|
||||
'X-ASBD-ID': asbdId,
|
||||
},
|
||||
body: new URLSearchParams(payload).toString(),
|
||||
parseResponse: (txt) => destr(txt.slice(9)), // remove "for (;;);"
|
||||
for (const el of dom.window.document.querySelectorAll('script[data-sjs]')) {
|
||||
try {
|
||||
const data = JSONPath({
|
||||
path: '$..user_id',
|
||||
json: JSON.parse(el.textContent || ''),
|
||||
});
|
||||
|
||||
if (data?.[0]) {
|
||||
return data[0];
|
||||
}
|
||||
} catch {
|
||||
// Skip invalid JSON
|
||||
}
|
||||
}
|
||||
|
||||
throw new NotFoundError('User ID not found');
|
||||
})
|
||||
.then((result): string => {
|
||||
if (!result || typeof result !== 'string') {
|
||||
throw new TypeError('Invalid user ID type');
|
||||
}
|
||||
return result;
|
||||
});
|
||||
|
||||
let userId = response.payload.payloads[pathName].result.exports.rootView.props.user_id;
|
||||
|
||||
if (!userId) {
|
||||
const fallbackResponse = await ofetch(instagramUrl(user), {
|
||||
headers: makeHeader(user, lsd),
|
||||
});
|
||||
|
||||
if (!fallbackResponse?.data?.user) {
|
||||
throw new NotFoundError('Instagram getUser API response is invalid');
|
||||
}
|
||||
|
||||
userId = fallbackResponse.data.user.id;
|
||||
if (!userId) {
|
||||
throw new NotFoundError('User ID not found in Instagram getUser API response');
|
||||
}
|
||||
}
|
||||
|
||||
return userId;
|
||||
});
|
||||
|
||||
const hasMedia = (post) => post.image_versions2 || post.carousel_media || post.video_versions;
|
||||
|
||||
const buildMedia = (post) => {
|
||||
let html = '';
|
||||
|
||||
|
|
@ -99,25 +138,13 @@ const buildMedia = (post) => {
|
|||
for (const media of post.carousel_media) {
|
||||
const firstImage = media.image_versions2?.candidates[0];
|
||||
const firstVideo = media.video_versions?.[0];
|
||||
if (firstVideo) {
|
||||
html += `<video controls autoplay loop poster="${firstImage.url}">`;
|
||||
html += `<source src="${firstVideo.url}"/>`;
|
||||
html += '</video>';
|
||||
} else {
|
||||
html += `<img src="${firstImage.url}"/>`;
|
||||
}
|
||||
html += firstVideo ? `<video controls autoplay loop poster="${firstImage.url}"><source src="${firstVideo.url}"/></video>` : `<img src="${firstImage.url}"/>`;
|
||||
}
|
||||
} else {
|
||||
const mainImage = post.image_versions2?.candidates?.[0];
|
||||
const mainVideo = post.video_versions?.[0];
|
||||
if (mainImage) {
|
||||
if (mainVideo) {
|
||||
html += `<video controls autoplay loop poster="${mainImage.url}">`;
|
||||
html += `<source src="${mainVideo.url}"/>`;
|
||||
html += '</video>';
|
||||
} else {
|
||||
html += `<img src="${mainImage.url}"/>`;
|
||||
}
|
||||
html += mainVideo ? `<video controls autoplay loop poster="${mainImage.url}"><source src="${mainVideo.url}"/></video>` : `<img src="${mainImage.url}"/>`;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -181,4 +208,4 @@ const buildContent = (item, options) => {
|
|||
return { title, description };
|
||||
};
|
||||
|
||||
export { apiUrl, profileUrl, threadUrl, THREADS_QUERY, REPLIES_QUERY, USER_AGENT, extractTokens, getUserId, makeHeader, hasMedia, buildMedia, buildContent };
|
||||
export { profileUrl, threadUrl, extractTokens, getUserId, buildContent };
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@
|
|||
"ip-regex": "5.0.0",
|
||||
"jsdom": "26.0.0",
|
||||
"json-bigint": "1.0.0",
|
||||
"jsonpath-plus": "^10.3.0",
|
||||
"jsrsasign": "10.9.0",
|
||||
"lru-cache": "11.0.2",
|
||||
"lz-string": "1.5.0",
|
||||
|
|
|
|||
|
|
@ -152,6 +152,9 @@ importers:
|
|||
json-bigint:
|
||||
specifier: 1.0.0
|
||||
version: 1.0.0
|
||||
jsonpath-plus:
|
||||
specifier: ^10.3.0
|
||||
version: 10.3.0
|
||||
jsrsasign:
|
||||
specifier: 10.9.0
|
||||
version: 10.9.0
|
||||
|
|
@ -1578,6 +1581,18 @@ packages:
|
|||
'@jridgewell/trace-mapping@0.3.25':
|
||||
resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==}
|
||||
|
||||
'@jsep-plugin/assignment@1.3.0':
|
||||
resolution: {integrity: sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ==}
|
||||
engines: {node: '>= 10.16.0'}
|
||||
peerDependencies:
|
||||
jsep: ^0.4.0||^1.0.0
|
||||
|
||||
'@jsep-plugin/regex@1.0.4':
|
||||
resolution: {integrity: sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg==}
|
||||
engines: {node: '>= 10.16.0'}
|
||||
peerDependencies:
|
||||
jsep: ^0.4.0||^1.0.0
|
||||
|
||||
'@lezer/common@1.2.3':
|
||||
resolution: {integrity: sha512-w7ojc8ejBqr2REPsWxJjrMFsA/ysDCFICn8zEOR9mrqzOu2amhITYuLD8ag6XZf0CFXDrhKqw7+tW8cX66NaDA==}
|
||||
|
||||
|
|
@ -4538,6 +4553,10 @@ packages:
|
|||
canvas:
|
||||
optional: true
|
||||
|
||||
jsep@1.4.0:
|
||||
resolution: {integrity: sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==}
|
||||
engines: {node: '>= 10.16.0'}
|
||||
|
||||
jsesc@3.0.2:
|
||||
resolution: {integrity: sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==}
|
||||
engines: {node: '>=6'}
|
||||
|
|
@ -4584,6 +4603,11 @@ packages:
|
|||
jsonfile@6.1.0:
|
||||
resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==}
|
||||
|
||||
jsonpath-plus@10.3.0:
|
||||
resolution: {integrity: sha512-8TNmfeTCk2Le33A3vRRwtuworG/L5RrgMvdjhKZxvyShO+mBu2fP50OWUjRLNtvw344DdDarFh9buFAZs5ujeA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
hasBin: true
|
||||
|
||||
jsonpointer@5.0.1:
|
||||
resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
|
@ -8059,6 +8083,14 @@ snapshots:
|
|||
'@jridgewell/resolve-uri': 3.1.2
|
||||
'@jridgewell/sourcemap-codec': 1.5.0
|
||||
|
||||
'@jsep-plugin/assignment@1.3.0(jsep@1.4.0)':
|
||||
dependencies:
|
||||
jsep: 1.4.0
|
||||
|
||||
'@jsep-plugin/regex@1.0.4(jsep@1.4.0)':
|
||||
dependencies:
|
||||
jsep: 1.4.0
|
||||
|
||||
'@lezer/common@1.2.3': {}
|
||||
|
||||
'@lezer/css@1.1.10':
|
||||
|
|
@ -11720,6 +11752,8 @@ snapshots:
|
|||
- supports-color
|
||||
- utf-8-validate
|
||||
|
||||
jsep@1.4.0: {}
|
||||
|
||||
jsesc@3.0.2: {}
|
||||
|
||||
jsesc@3.1.0: {}
|
||||
|
|
@ -11752,6 +11786,12 @@ snapshots:
|
|||
optionalDependencies:
|
||||
graceful-fs: 4.2.11
|
||||
|
||||
jsonpath-plus@10.3.0:
|
||||
dependencies:
|
||||
'@jsep-plugin/assignment': 1.3.0(jsep@1.4.0)
|
||||
'@jsep-plugin/regex': 1.0.4(jsep@1.4.0)
|
||||
jsep: 1.4.0
|
||||
|
||||
jsonpointer@5.0.1: {}
|
||||
|
||||
jsprim@1.4.2:
|
||||
|
|
|
|||
Loading…
Reference in New Issue