From 1bc24075c75a546b7fe2c2944da0b02023ca1b0f Mon Sep 17 00:00:00 2001 From: DIYgod Date: Tue, 19 Mar 2024 22:11:20 +0800 Subject: [PATCH] feat(twitter): list for webapi --- lib/routes/twitter/api-fallback-common.ts | 17 ---- lib/routes/twitter/api/index.ts | 2 + lib/routes/twitter/api/web-api/api.ts | 15 ++++ lib/routes/twitter/api/web-api/constants.ts | 23 ++++++ lib/routes/twitter/collection.ts | 86 --------------------- lib/routes/twitter/list.ts | 55 ++++++------- lib/routes/twitter/media.ts | 2 +- 7 files changed, 65 insertions(+), 135 deletions(-) delete mode 100644 lib/routes/twitter/api-fallback-common.ts delete mode 100644 lib/routes/twitter/collection.ts diff --git a/lib/routes/twitter/api-fallback-common.ts b/lib/routes/twitter/api-fallback-common.ts deleted file mode 100644 index e08e04cf8..000000000 --- a/lib/routes/twitter/api-fallback-common.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { config } from '@/config'; -import logger from '@/utils/logger'; -import utils from './utils'; - -export default async (ctx, devApiImpl, webApiImpl) => { - const { force_web_api } = utils.parseRouteParams(ctx.req.param('routeParams')); - - if (!force_web_api && config.twitter && config.twitter.consumer_key && config.twitter.consumer_secret) { - try { - await devApiImpl(ctx); - return; - } catch (error) { - logger.error(`Fallback to Twitter web API due to developer API error:\n${error.stack}`); - } - } - await webApiImpl(ctx); -}; diff --git a/lib/routes/twitter/api/index.ts b/lib/routes/twitter/api/index.ts index 209013c79..a7d885f9c 100644 --- a/lib/routes/twitter/api/index.ts +++ b/lib/routes/twitter/api/index.ts @@ -15,6 +15,7 @@ let api: { getUserLikes: ApiItem; getUserTweet: ApiItem; getSearch: ApiItem; + getList: ApiItem; } = { init: () => { throw new Error('Twitter API is not configured'); @@ -26,6 +27,7 @@ let api: { getUserLikes: () => null, getUserTweet: () => null, getSearch: () => null, + getList: () => null, }; if (enableWebApi) { diff --git a/lib/routes/twitter/api/web-api/api.ts b/lib/routes/twitter/api/web-api/api.ts index 2a280d789..6063b1cc4 100644 --- a/lib/routes/twitter/api/web-api/api.ts +++ b/lib/routes/twitter/api/web-api/api.ts @@ -130,6 +130,20 @@ const getSearch = async (keywords: string, params?: Record) => ) ); +const getList = async (id: string, params?: Record) => + gatherLegacyFromData( + await paginationTweets( + 'ListLatestTweetsTimeline', + undefined, + { + ...params, + listId: id, + count: 20, + }, + ['list', 'tweets_timeline', 'timeline'] + ) + ); + const getUser = async (id: string) => { const userData: any = await getUserData(id); return (userData.data?.user || userData.data?.user_result)?.result?.legacy; @@ -143,5 +157,6 @@ export default { getUserLikes, getUserTweet, getSearch, + getList, init: () => {}, }; diff --git a/lib/routes/twitter/api/web-api/constants.ts b/lib/routes/twitter/api/web-api/constants.ts index 436f2d3fb..0982fc6a2 100644 --- a/lib/routes/twitter/api/web-api/constants.ts +++ b/lib/routes/twitter/api/web-api/constants.ts @@ -9,6 +9,7 @@ const graphQLEndpointsPlain = [ '/graphql/B8I_QCljDBVfin21TTWMqA/Likes', '/graphql/tD8zKvQzwY3kdx5yz6YmOw/UserByRestId', '/graphql/flaR-PUMshxFWZWPNpq4zA/SearchTimeline', + '/graphql/TOTgqavWmxywKv5IbMMK1w/ListLatestTweetsTimeline', ]; const gqlMap = Object.fromEntries(graphQLEndpointsPlain.map((endpoint) => [endpoint.split('/')[3].replace(/V2$|Query$|QueryV2$/, ''), endpoint])); @@ -106,6 +107,28 @@ const gqlFeatures = { longform_notetweets_inline_media_enabled: true, responsive_web_enhance_cards_enabled: false, }, + ListLatestTweetsTimeline: { + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + c9s_tweet_anatomy_moderator_badge_enabled: true, + tweetypie_unmention_optimization_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + tweet_awards_web_tipping_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + rweb_video_timestamps_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_enhance_cards_enabled: false, + }, }; const timelineParams = { diff --git a/lib/routes/twitter/collection.ts b/lib/routes/twitter/collection.ts deleted file mode 100644 index 9d2ea78d0..000000000 --- a/lib/routes/twitter/collection.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { Route } from '@/types'; -import utils from './utils'; -import { config } from '@/config'; -const T = {}; -import { TwitterApi } from 'twitter-api-v2'; -import { fallback, queryToBoolean } from '@/utils/readable-social'; - -export const route: Route = { - path: '/collection/:uid/:collectionId/:routeParams?', - categories: ['social-media'], - example: '/twitter/collection/DIYgod/1527857429467172864', - parameters: { uid: 'username, should match the generated token', collectionId: 'collection ID, can be found in URL', routeParams: 'extra parameters, see the table above' }, - features: { - requireConfig: [ - { - name: 'TWITTER_USERNAME', - description: '', - }, - { - name: 'TWITTER_PASSWORD', - description: '', - }, - ], - requirePuppeteer: false, - antiCrawler: false, - supportBT: false, - supportPodcast: false, - supportScihub: false, - }, - name: 'Collection', - maintainers: ['TonyRL'], - handler, - description: `:::warning - This route requires Twitter token's corresponding id, therefore it's only available when self-hosting, refer to the [Deploy Guide](/install/#route-specific-configurations) for route-specific configurations. - :::`, -}; - -async function handler(ctx) { - const uid = ctx.req.param('uid'); - const collectionId = ctx.req.param('collectionId'); - const cookie = config.twitter.tokens[uid]; - if (!cookie) { - throw new Error(`lacking twitter token for user ${uid}`); - } else if (!T[uid]) { - const token = cookie.split(','); - T[uid] = new TwitterApi({ - appKey: token[0], - appSecret: token[1], - accessToken: token[2], - accessSecret: token[3], - }).readOnly; - } - - const id = `custom-${collectionId}`; - - const data = await T[uid].v1.get('collections/entries.json', { - id, - count: ctx.req.query('limit') ? (Number(ctx.req.query('limit')) > 200 ? 200 : Number(ctx.req.query('limit'))) : 200, - }); - - const routeParams = new URLSearchParams(ctx.req.param('routeParams')); - - // fix user without screen_name - for (const tweet of Object.values(data.objects.tweets)) { - tweet.user = data.objects.users[tweet.user.id_str]; - if (tweet.quoted_status) { - tweet.quoted_status.user = data.objects.users[tweet.quoted_status.user.id_str]; - } - } - - return { - title: data.objects.timelines[id].name, - description: data.objects.timelines[id].description, - link: data.objects.timelines[id].collection_url, - item: utils.ProcessFeed( - ctx, - { - data: Object.values(data.objects.tweets), - }, - { - showAuthorInTitle: fallback(undefined, queryToBoolean(routeParams.get('showAuthorInTitle')), true), - showAuthorInDesc: fallback(undefined, queryToBoolean(routeParams.get('showAuthorInDesc')), true), - } - ), - }; -} diff --git a/lib/routes/twitter/list.ts b/lib/routes/twitter/list.ts index 2abbb1372..79cdea76e 100644 --- a/lib/routes/twitter/list.ts +++ b/lib/routes/twitter/list.ts @@ -1,15 +1,27 @@ import { Route } from '@/types'; -import cache from '@/utils/cache'; +import api from './api'; import utils from './utils'; -import { config } from '@/config'; export const route: Route = { - path: '/list/:id/:name/:routeParams?', + path: '/list/:id/:routeParams?', categories: ['social-media'], example: '/twitter/list/ladyleet/javascript', parameters: { id: 'username', name: 'list name', routeParams: 'extra parameters, see the table above' }, features: { - requireConfig: false, + requireConfig: [ + { + name: 'TWITTER_USERNAME', + description: '', + }, + { + name: 'TWITTER_PASSWORD', + description: '', + }, + { + name: 'TWITTER_COOKIE', + description: '', + }, + ], requirePuppeteer: false, antiCrawler: false, supportBT: false, @@ -17,40 +29,21 @@ export const route: Route = { supportScihub: false, }, name: 'List timeline', - maintainers: ['xyqfer'], + maintainers: ['DIYgod', 'xyqfer'], handler, }; async function handler(ctx) { - if (!config.twitter || !config.twitter.consumer_key || !config.twitter.consumer_secret) { - throw new Error('Twitter RSS is disabled due to the lack of relevant config'); - } - const { id, name } = ctx.req.param(); - const client = await utils.getAppClient(); + const id = ctx.req.param('id'); + const { count } = utils.parseRouteParams(ctx.req.param('routeParams')); + const params = count ? { count } : {}; - const list_data = await cache.tryGet(`twitter_lists_list_screen_name:${id}`, async () => { - const data = await client.v1.get('lists/list.json', { - screen_name: id, - }); - - const cached_lists = {}; - for (const e of data) { - cached_lists[e.name] = { id: e.id_str, slug: e.slug }; - } - - return cached_lists; - }); - const cur_list = list_data[name]; - - const data = await client.v1.get('lists/statuses.json', { - list_id: cur_list.id, - slug: cur_list.slug, - tweet_mode: 'extended', - }); + await api.init(); + const data = await api.getList(id, params); return { - title: `Twitter List - ${id}/${name}`, - link: `https://twitter.com/${id}/lists/${name}`, + title: `Twitter List - ${id}`, + link: `https://twitter.com/i/lists/${id}`, item: utils.ProcessFeed(ctx, { data, }), diff --git a/lib/routes/twitter/media.ts b/lib/routes/twitter/media.ts index f88762451..160c747e4 100644 --- a/lib/routes/twitter/media.ts +++ b/lib/routes/twitter/media.ts @@ -29,7 +29,7 @@ export const route: Route = { supportScihub: false, }, name: 'User media', - maintainers: ['yindaheng98', 'Rongronggg9'], + maintainers: ['DIYgod', 'yindaheng98', 'Rongronggg9'], handler, radar: [ {