From 52edc7e393438631d09f77e8f55960f261d266bb Mon Sep 17 00:00:00 2001 From: DIYgod Date: Tue, 19 Mar 2024 21:41:39 +0800 Subject: [PATCH] feat(twitter): media for webapi --- lib/routes/twitter/api/web-api/api.ts | 36 ++++++++++++++++++--- lib/routes/twitter/api/web-api/constants.ts | 31 ++++++++++++------ lib/routes/twitter/api/web-api/utils.ts | 16 +++++---- lib/routes/twitter/media.ts | 4 +++ lib/routes/twitter/user.ts | 4 +++ 5 files changed, 72 insertions(+), 19 deletions(-) diff --git a/lib/routes/twitter/api/web-api/api.ts b/lib/routes/twitter/api/web-api/api.ts index 8c81c8738..709031a05 100644 --- a/lib/routes/twitter/api/web-api/api.ts +++ b/lib/routes/twitter/api/web-api/api.ts @@ -1,4 +1,4 @@ -import { baseUrl, gqlMap, gqlFeatures, gqlFieldToggles } from './constants'; +import { baseUrl, gqlMap, gqlFeatures } from './constants'; import { config } from '@/config'; import cache from '@/utils/cache'; import { twitterGot, paginationTweets, gatherLegacyFromData } from './utils'; @@ -12,7 +12,9 @@ const getUserData = (id) => withSafetyModeUserFields: true, }), features: JSON.stringify(gqlFeatures.UserByRestId), - fieldToggles: JSON.stringify(gqlFieldToggles.UserByScreenName), + fieldToggles: JSON.stringify({ + withAuxiliaryUserLabels: false, + }), }); } return twitterGot(`${baseUrl}${gqlMap.UserByScreenName}`, { @@ -21,7 +23,9 @@ const getUserData = (id) => withSafetyModeUserFields: true, }), features: JSON.stringify(gqlFeatures.UserByScreenName), - fieldToggles: JSON.stringify(gqlFieldToggles.UserByScreenName), + fieldToggles: JSON.stringify({ + withAuxiliaryUserLabels: false, + }), }); }); @@ -62,7 +66,31 @@ const getUserTweetsAndReplies = (id: string, params?: Record) => ) ); -const getUserMedia = (id: string, params?: Record) => cacheTryGet(id, params, async (id, params = {}) => gatherLegacyFromData(await paginationTweets('MediaTimeline', id, params))); +const getUserMedia = (id: string, params?: Record) => + cacheTryGet(id, params, async (id, params = {}) => { + const cursorSource = await paginationTweets('UserMedia', id, { + ...params, + count: 20, + includePromotedContent: false, + withClientEventToken: false, + withBirdwatchNotes: false, + withVoice: true, + withV2Timeline: true, + }); + const cursor = cursorSource.find((i) => i.content?.cursorType === 'Top').content.value; + return gatherLegacyFromData( + await paginationTweets('UserMedia', id, { + ...params, + cursor, + count: 20, + includePromotedContent: false, + withClientEventToken: false, + withBirdwatchNotes: false, + withVoice: true, + withV2Timeline: true, + }) + ); + }); const getUserLikes = (id: string, params?: Record) => cacheTryGet(id, params, async (id, params = {}) => gatherLegacyFromData(await paginationTweets('Likes', id, params))); diff --git a/lib/routes/twitter/api/web-api/constants.ts b/lib/routes/twitter/api/web-api/constants.ts index 01df27da1..f6fa059c2 100644 --- a/lib/routes/twitter/api/web-api/constants.ts +++ b/lib/routes/twitter/api/web-api/constants.ts @@ -61,14 +61,27 @@ const gqlFeatures = { longform_notetweets_inline_media_enabled: true, responsive_web_enhance_cards_enabled: false, }, -}; - -const gqlFieldToggles = { - UserByScreenName: { - withAuxiliaryUserLabels: false, - }, - UserByRestId: { - withAuxiliaryUserLabels: false, + UserMedia: { + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + c9s_tweet_anatomy_moderator_badge_enabled: true, + tweetypie_unmention_optimization_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + tweet_awards_web_tipping_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + rweb_video_timestamps_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_enhance_cards_enabled: false, }, }; @@ -90,4 +103,4 @@ const timelineParams = { const bearerToken = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; -export { baseUrl, gqlMap, gqlFeatures, gqlFieldToggles, timelineParams, bearerToken }; +export { baseUrl, gqlMap, gqlFeatures, timelineParams, bearerToken }; diff --git a/lib/routes/twitter/api/web-api/utils.ts b/lib/routes/twitter/api/web-api/utils.ts index 5f89ede02..dce219237 100644 --- a/lib/routes/twitter/api/web-api/utils.ts +++ b/lib/routes/twitter/api/web-api/utils.ts @@ -12,7 +12,7 @@ export const twitterGot = async (url, params) => { config.twitter.cookie .split(';') .map((c) => Cookie.parse(c)?.toJSON()) - .map((c) => [c.key, c.value]) + .map((c) => [c?.key, c?.value]) ); if (!jsonCookie || !jsonCookie.auth_token || !jsonCookie.ct0) { throw new Error('Twitter cookie is not valid'); @@ -66,17 +66,21 @@ export const paginationTweets = async (endpoint: string, userId: number | undefi instructions = data.user.result.timeline_v2.timeline.instructions; } - return instructions.find((i) => i.__typename === 'TimelineAddEntries' || i.type === 'TimelineAddEntries').entries; + const entries1 = instructions.find((i) => i.type === 'TimelineAddToModule')?.moduleItems; // Media + const entries2 = instructions.find((i) => i.type === 'TimelineAddEntries').entries; + return entries1 || entries2; }; -export function gatherLegacyFromData(entries, filterNested?: string[], userId?: number | string) { - const tweets = []; - const filteredEntries = []; +export function gatherLegacyFromData(entries: any[], filterNested?: string[], userId?: number | string) { + const tweets: any[] = []; + const filteredEntries: any[] = []; for (const entry of entries) { const entryId = entry.entryId; if (entryId) { if (entryId.startsWith('tweet-')) { filteredEntries.push(entry); + } else if (entryId.startsWith('profile-grid-0-tweet-')) { + filteredEntries.push(entry); } if (filterNested && filterNested.some((f) => entryId.startsWith(f))) { filteredEntries.push(...entry.content.items); @@ -98,7 +102,7 @@ export function gatherLegacyFromData(entries, filterNested?: string[], userId?: } t.legacy.user = t.core?.user_result?.result?.legacy || t.core?.user_results?.result?.legacy; t.legacy.id_str = t.rest_id; // avoid falling back to conversation_id_str elsewhere - const quote = t.quoted_status_result?.result; + const quote = t.quoted_status_result?.result?.tweet || t.quoted_status_result?.result; if (quote) { t.legacy.quoted_status = quote.legacy; t.legacy.quoted_status.user = quote.core.user_result?.result?.legacy || quote.core.user_results?.result?.legacy; diff --git a/lib/routes/twitter/media.ts b/lib/routes/twitter/media.ts index 3a184f348..4e5d79720 100644 --- a/lib/routes/twitter/media.ts +++ b/lib/routes/twitter/media.ts @@ -17,6 +17,10 @@ export const route: Route = { name: 'TWITTER_PASSWORD', description: '', }, + { + name: 'TWITTER_COOKIE', + description: '', + }, ], requirePuppeteer: false, antiCrawler: false, diff --git a/lib/routes/twitter/user.ts b/lib/routes/twitter/user.ts index 75f548e44..e2aac02b9 100644 --- a/lib/routes/twitter/user.ts +++ b/lib/routes/twitter/user.ts @@ -21,6 +21,10 @@ export const route: Route = { name: 'TWITTER_PASSWORD', description: '', }, + { + name: 'TWITTER_COOKIE', + description: '', + }, ], requirePuppeteer: false, antiCrawler: false,