xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(twitter): api structure and web user

This commit is contained in:
DIYgod 2024-03-19 20:57:19 +08:00
parent 95ace09917
commit 4a7c00f274
No known key found for this signature in database
20 changed files with 471 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/config.ts
View File

@ -252,6 +252,7 @@ export type Config = {
oauthTokenSecrets?: string[];
username?: string;
password?: string;
cookie?: string;
};
weibo: {
app_key?: string;
@ -591,7 +592,7 @@ const calculateValue = () => {
oauthTokenSecrets: envs.TWITTER_OAUTH_TOKEN_SECRET?.split(','),
username: envs.TWITTER_USERNAME,
password: envs.TWITTER_PASSWORD,
authenticationSecret: envs.TWITTER_AUTHENTICATION_SECRET,
cookie: envs.TWITTER_COOKIE,
},
weibo: {
app_key: envs.WEIBO_APP_KEY,

2
lib/routes/twitter/developer-api/search.ts → lib/routes/twitter/api/developer-api/search.ts
View File

@ -1,4 +1,4 @@
import utils from '../utils';
import utils from '../../utils';
export default async (ctx) => {
const keyword = ctx.req.param('keyword');

2
lib/routes/twitter/developer-api/user.ts → lib/routes/twitter/api/developer-api/user.ts
View File

@ -1,4 +1,4 @@
import utils from '../utils';
import utils from '../../utils';
export default async (ctx) => {
const id = ctx.req.param('id');

37
lib/routes/twitter/api/index.ts Normal file
View File

@ -0,0 +1,37 @@
import mobileApi from './mobile-api/api';
import webApi from './web-api/api';
import { config } from '@/config';
const enableMobileApi = config.twitter.username && config.twitter.password;
const enableWebApi = config.twitter.cookie;
type ApiItem = (id: string, params?: Record<string, any>) => Promise<Record<string, any>> | Record<string, any> | null;
let api: {
init: () => void;
getUser: ApiItem;
getUserTweets: ApiItem;
getUserTweetsAndReplies: ApiItem;
getUserMedia: ApiItem;
getUserLikes: ApiItem;
getUserTweet: ApiItem;
getSearch: ApiItem;
} = {
init: () => {
throw new Error('Twitter API is not configured');
},
getUser: () => null,
getUserTweets: () => null,
getUserTweetsAndReplies: () => null,
getUserMedia: () => null,
getUserLikes: () => null,
getUserTweet: () => null,
getSearch: () => null,
};
if (enableWebApi) {
api = webApi;
} else if (enableMobileApi) {
api = mobileApi;
}
export default api;

5
lib/routes/twitter/mobile-api/twitter-api.ts → lib/routes/twitter/api/mobile-api/api.ts
View File

@ -5,7 +5,7 @@ import got from '@/utils/got';
import OAuth from 'oauth-1.0a';
import CryptoJS from 'crypto-js';
import queryString from 'query-string';
import { getToken } from './token';
import { initToken, getToken } from './token';
import cache from '@/utils/cache';
const twitterGot = async (url, params) => {
@ -269,7 +269,7 @@ const getUserTweet = (id, params) => cacheTryGet(id, params, getUserTweetByStatu
const getSearch = async (keywords, params = {}) => gatherLegacyFromData(await timelineKeywords(keywords, params));
export {
export default {
getUser,
getUserTweets,
getUserTweetsAndReplies,
@ -278,4 +278,5 @@ export {
excludeRetweet,
getSearch,
getUserTweet,
init: initToken,
};

0
lib/routes/twitter/mobile-api/constants.ts → lib/routes/twitter/api/mobile-api/constants.ts
View File

0
lib/routes/twitter/mobile-api/login.ts → lib/routes/twitter/api/mobile-api/login.ts
View File

0
lib/routes/twitter/mobile-api/token.ts → lib/routes/twitter/api/mobile-api/token.ts
View File

121
lib/routes/twitter/api/web-api/api.ts Normal file
View File

@ -0,0 +1,121 @@
import { baseUrl, gqlMap, gqlFeatures, gqlFieldToggles } from './constants';
import { config } from '@/config';
import cache from '@/utils/cache';
import { twitterGot, paginationTweets, gatherLegacyFromData } from './utils';
const getUserData = (id) =>
cache.tryGet(`twitter-userdata-${id}`, () => {
if (id.startsWith('+')) {
return twitterGot(`${baseUrl}${gqlMap.UserByRestId}`, {
variables: JSON.stringify({
userId: id.slice(1),
withSafetyModeUserFields: true,
}),
features: JSON.stringify(gqlFeatures.UserByRestId),
fieldToggles: JSON.stringify(gqlFieldToggles.UserByScreenName),
});
}
return twitterGot(`${baseUrl}${gqlMap.UserByScreenName}`, {
variables: JSON.stringify({
screen_name: id,
withSafetyModeUserFields: true,
}),
features: JSON.stringify(gqlFeatures.UserByScreenName),
fieldToggles: JSON.stringify(gqlFieldToggles.UserByScreenName),
});
});
const cacheTryGet = async (_id, params, func) => {
const userData: any = await getUserData(_id);
const id = (userData.data?.user || userData.data?.user_result)?.result?.rest_id;
if (id === undefined) {
throw new Error('User not found');
}
const funcName = func.name;
const paramsString = JSON.stringify(params);
return cache.tryGet(`twitter:${id}:${funcName}:${paramsString}`, () => func(id, params), config.cache.routeExpire, false);
};
const getUserTweets = (id: string, params?: Record<string, any>) =>
cacheTryGet(id, params, async (id, params = {}) =>
gatherLegacyFromData(
await paginationTweets('UserTweets', id, {
...params,
withQuickPromoteEligibilityTweetFields: true,
})
)
);
const getUserTweetsAndReplies = (id: string, params?: Record<string, any>) =>
cacheTryGet(id, params, async (id, params = {}) =>
gatherLegacyFromData(
await paginationTweets('UserTweetsAndReplies', id, {
...params,
count: 20,
includePromotedContent: true,
withCommunity: true,
withVoice: true,
withV2Timeline: true,
}),
['profile-conversation-'],
id
)
);
const getUserMedia = (id: string, params?: Record<string, any>) => cacheTryGet(id, params, async (id, params = {}) => gatherLegacyFromData(await paginationTweets('MediaTimeline', id, params)));
const getUserLikes = (id: string, params?: Record<string, any>) => cacheTryGet(id, params, async (id, params = {}) => gatherLegacyFromData(await paginationTweets('Likes', id, params)));
const getUserTweet = (id: string, params?: Record<string, any>) =>
cacheTryGet(id, params, async (id, params = {}) =>
gatherLegacyFromData(
await paginationTweets(
'TweetDetail',
id,
{
...params,
includeHasBirdwatchNotes: false,
includePromotedContent: false,
withBirdwatchNotes: false,
withVoice: false,
withV2Timeline: true,
},
['threaded_conversation_with_injections_v2']
),
['homeConversation-', 'conversationthread-']
)
);
const getSearch = async (keywords: string, params?: Record<string, any>) =>
gatherLegacyFromData(
await paginationTweets(
'SearchTimeline',
undefined,
{
...params,
rawQuery: keywords,
count: 20,
product: 'Latest',
withDownvotePerspective: false,
withReactionsMetadata: false,
withReactionsPerspective: false,
},
['search_by_raw_query', 'search_timeline', 'timeline']
)
);
const getUser = async (id: string) => {
const userData: any = await getUserData(id);
return (userData.data?.user || userData.data?.user_result)?.result?.legacy;
};
export default {
getUser,
getUserTweets,
getUserTweetsAndReplies,
getUserMedia,
getUserLikes,
getUserTweet,
getSearch,
init: () => {},
};

93
lib/routes/twitter/api/web-api/constants.ts Normal file
View File

@ -0,0 +1,93 @@
const baseUrl = 'https://twitter.com/i/api';
const graphQLEndpointsPlain = [
'/graphql/eS7LO5Jy3xgmd3dbL044EA/UserTweets',
'/graphql/k5XapwcSikNsEsILW5FvgA/UserByScreenName',
'/graphql/k3YiLNE_MAy5J-NANLERdg/HomeTimeline',
'/graphql/3GeIaLmNhTm1YsUmxR57tg/UserTweetsAndReplies',
'/graphql/TOU4gQw8wXIqpSzA4TYKgg/UserMedia',
'/graphql/B8I_QCljDBVfin21TTWMqA/Likes',
'/graphql/tD8zKvQzwY3kdx5yz6YmOw/UserByRestId',
];
const gqlMap = Object.fromEntries(graphQLEndpointsPlain.map((endpoint) => [endpoint.split('/')[3].replace(/V2$|Query$|QueryV2$/, ''), endpoint]));
const gqlFeatures = {
UserByScreenName: {
hidden_profile_likes_enabled: true,
hidden_profile_subscriptions_enabled: true,
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
subscriptions_verification_info_is_identity_verified_enabled: true,
subscriptions_verification_info_verified_since_enabled: true,
highlights_tweets_tab_ui_enabled: true,
responsive_web_twitter_article_notes_tab_enabled: true,
creator_subscriptions_tweet_preview_api_enabled: true,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
responsive_web_graphql_timeline_navigation_enabled: true,
},
UserByRestId: {
hidden_profile_likes_enabled: true,
hidden_profile_subscriptions_enabled: true,
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
subscriptions_verification_info_is_identity_verified_enabled: true,
subscriptions_verification_info_verified_since_enabled: true,
highlights_tweets_tab_ui_enabled: true,
responsive_web_twitter_article_notes_tab_enabled: true,
creator_subscriptions_tweet_preview_api_enabled: true,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
responsive_web_graphql_timeline_navigation_enabled: true,
},
UserTweetsAndReplies: {
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
creator_subscriptions_tweet_preview_api_enabled: true,
responsive_web_graphql_timeline_navigation_enabled: true,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
c9s_tweet_anatomy_moderator_badge_enabled: true,
tweetypie_unmention_optimization_enabled: true,
responsive_web_edit_tweet_api_enabled: true,
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
view_counts_everywhere_api_enabled: true,
longform_notetweets_consumption_enabled: true,
responsive_web_twitter_article_tweet_consumption_enabled: true,
tweet_awards_web_tipping_enabled: false,
freedom_of_speech_not_reach_fetch_enabled: true,
standardized_nudges_misinfo: true,
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
rweb_video_timestamps_enabled: true,
longform_notetweets_rich_text_read_enabled: true,
longform_notetweets_inline_media_enabled: true,
responsive_web_enhance_cards_enabled: false,
},
};
const gqlFieldToggles = {
UserByScreenName: {
withAuxiliaryUserLabels: false,
},
UserByRestId: {
withAuxiliaryUserLabels: false,
},
};
const timelineParams = {
include_can_media_tag: 1,
include_cards: 1,
include_entities: 1,
include_profile_interstitial_type: 0,
include_quote_count: 0,
include_reply_count: 0,
include_user_entities: 0,
include_ext_reply_count: 0,
include_ext_media_color: 0,
cards_platform: 'Web-13',
tweet_mode: 'extended',
send_error_codes: 1,
simple_quoted_tweet: 1,
};
const bearerToken = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
export { baseUrl, gqlMap, gqlFeatures, gqlFieldToggles, timelineParams, bearerToken };

120
lib/routes/twitter/api/web-api/utils.ts Normal file
View File

@ -0,0 +1,120 @@
import { baseUrl, gqlFeatures, bearerToken, gqlMap } from './constants';
import { config } from '@/config';
import got from '@/utils/got';
import queryString from 'query-string';
import { Cookie } from 'tough-cookie';
export const twitterGot = async (url, params) => {
if (!config.twitter.cookie) {
throw new Error('Twitter cookie is not configured');
}
const jsonCookie = Object.fromEntries(
config.twitter.cookie
.split(';')
.map((c) => Cookie.parse(c)?.toJSON())
.map((c) => [c.key, c.value])
);
if (!jsonCookie || !jsonCookie.auth_token || !jsonCookie.ct0) {
throw new Error('Twitter cookie is not valid');
}
const requestData = {
url: `${url}?${queryString.stringify(params)}`,
method: 'GET',
headers: {
authority: 'twitter.com',
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
authorization: bearerToken,
'cache-control': 'no-cache',
'content-type': 'application/json',
cookie: config.twitter.cookie,
dnt: '1',
pragma: 'no-cache',
referer: 'https://twitter.com/narendramodi',
'x-csrf-token': jsonCookie.ct0,
'x-twitter-active-user': 'yes',
'x-twitter-auth-type': 'OAuth2Session',
'x-twitter-client-language': 'en',
},
};
const response = await got(requestData.url, {
headers: requestData.headers,
});
return response.data;
};
export const paginationTweets = async (endpoint: string, userId: number | undefined, variables: Record<string, any>, path?: string[]) => {
const { data } = await twitterGot(baseUrl + gqlMap[endpoint], {
variables: JSON.stringify({
...variables,
userId,
}),
features: JSON.stringify(gqlFeatures[endpoint]),
});
let instructions;
if (path) {
instructions = data;
for (const p of path) {
instructions = instructions[p];
}
instructions = instructions.instructions;
} else {
instructions = data.user.result.timeline_v2.timeline.instructions;
}
return instructions.find((i) => i.__typename === 'TimelineAddEntries' || i.type === 'TimelineAddEntries').entries;
};
export function gatherLegacyFromData(entries, filterNested?: string[], userId?: number | string) {
const tweets = [];
const filteredEntries = [];
for (const entry of entries) {
const entryId = entry.entryId;
if (entryId) {
if (entryId.startsWith('tweet-')) {
filteredEntries.push(entry);
}
if (filterNested && filterNested.some((f) => entryId.startsWith(f))) {
filteredEntries.push(...entry.content.items);
}
}
}
for (const entry of filteredEntries) {
if (entry.entryId) {
const content = entry.content || entry.item;
let tweet = content?.content?.tweetResult?.result || content?.itemContent?.tweet_results?.result;
if (tweet && tweet.tweet) {
tweet = tweet.tweet;
}
if (tweet) {
const retweet = tweet.legacy?.retweeted_status_result?.result;
for (const t of [tweet, retweet]) {
if (!t?.legacy) {
continue;
}
t.legacy.user = t.core?.user_result?.result?.legacy || t.core?.user_results?.result?.legacy;
t.legacy.id_str = t.rest_id; // avoid falling back to conversation_id_str elsewhere
const quote = t.quoted_status_result?.result;
if (quote) {
t.legacy.quoted_status = quote.legacy;
t.legacy.quoted_status.user = quote.core.user_result?.result?.legacy || quote.core.user_results?.result?.legacy;
}
}
const legacy = tweet.legacy;
if (legacy) {
if (retweet) {
legacy.retweeted_status = retweet.legacy;
}
if (userId === undefined || legacy.user_id_str === userId + '') {
tweets.push(legacy);
}
}
}
}
}
return tweets;
}

16
lib/routes/twitter/keyword.ts
View File

@ -1,5 +1,6 @@
import { Route } from '@/types';
import webApiImpl from './mobile-api/search';
import api from './api';
import utils from './utils';
export const route: Route = {
path: '/keyword/:keyword/:routeParams?',
@ -34,5 +35,16 @@ export const route: Route = {
};
async function handler(ctx) {
return await webApiImpl(ctx);
const keyword = ctx.req.param('keyword');
await api.init();
const data = await api.getSearch(keyword);
return {
title: `Twitter Keyword - ${keyword}`,
link: `https://twitter.com/search?q=${encodeURIComponent(keyword)}`,
item: utils.ProcessFeed(ctx, {
data,
}),
allowEmpty: true,
};
}

22
lib/routes/twitter/media.ts
View File

@ -1,5 +1,6 @@
import { Route } from '@/types';
import webApiImpl from './mobile-api/media';
import api from './api';
import utils from './utils';
export const route: Route = {
path: '/media/:id/:routeParams?',
@ -35,5 +36,22 @@ export const route: Route = {
};
async function handler(ctx) {
return await webApiImpl(ctx);
const id = ctx.req.param('id');
const { count } = utils.parseRouteParams(ctx.req.param('routeParams'));
const params = count ? { count } : {};
await api.init();
const userInfo = await api.getUser(id);
const data = await api.getUserMedia(id, params);
const profileImageUrl = userInfo.profile_image_url || userInfo.profile_image_url_https;
return {
title: `Twitter @${userInfo.name}`,
link: `https://twitter.com/${userInfo.screen_name}/media`,
image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
description: userInfo.description,
item: utils.ProcessFeed(ctx, {
data,
}),
};
}

25
lib/routes/twitter/mobile-api/media.ts
View File

@ -1,25 +0,0 @@
import utils from '../utils';
// import { config } from '@/config';
import { getUser, getUserMedia } from './twitter-api';
import { initToken } from './token';
export default async (ctx) => {
const id = ctx.req.param('id');
const { count } = utils.parseRouteParams(ctx.req.param('routeParams'));
const params = count ? { count } : {};
await initToken();
const userInfo = await getUser(id);
const data = await getUserMedia(id, params);
const profileImageUrl = userInfo.profile_image_url || userInfo.profile_image_url_https;
return {
title: `Twitter @${userInfo.name}`,
link: `https://twitter.com/${userInfo.screen_name}/media`,
image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
description: userInfo.description,
item: utils.ProcessFeed(ctx, {
data,
}),
};
};

18
lib/routes/twitter/mobile-api/search.ts
View File

@ -1,18 +0,0 @@
import utils from '../utils';
import { getSearch } from './twitter-api';
import { initToken } from './token';
export default async (ctx) => {
const keyword = ctx.req.param('keyword');
await initToken();
const data = await getSearch(keyword);
return {
title: `Twitter Keyword - ${keyword}`,
link: `https://twitter.com/search?q=${encodeURIComponent(keyword)}`,
item: utils.ProcessFeed(ctx, {
data,
}),
allowEmpty: true,
};
};

27
lib/routes/twitter/mobile-api/tweet.ts
View File

@ -1,27 +0,0 @@
// import { config } from '@/config';
import { getUser, getUserTweet } from './twitter-api';
import utils from '../utils';
import { fallback, queryToBoolean } from '@/utils/readable-social';
import { config } from '@/config';
import { initToken } from './token';
export default async (ctx) => {
const id = ctx.req.param('id');
const status = ctx.req.param('status');
const routeParams = new URLSearchParams(ctx.req.param('original'));
const original = fallback(undefined, queryToBoolean(routeParams.get('original')), false);
const params = { focalTweetId: status };
await initToken();
const userInfo = await getUser(id);
const data = await getUserTweet(id, params);
const profileImageUrl = userInfo.profile_image_url || userInfo.profile_image_url_https;
const item = original && config.isPackage ? data : utils.ProcessFeed(ctx, { data });
return {
title: `Twitter @${userInfo.name}`,
link: `https://twitter.com/${userInfo.screen_name}/status/${status}`,
image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
description: userInfo.description,
item,
};
};

30
lib/routes/twitter/mobile-api/user.ts
View File

@ -1,30 +0,0 @@
import utils from '../utils';
import { getUser, getUserTweets, getUserTweetsAndReplies, excludeRetweet } from './twitter-api';
import { initToken } from './token';
export default async (ctx) => {
const id = ctx.req.param('id');
// For compatibility
const { count, exclude_replies, include_rts } = utils.parseRouteParams(ctx.req.param('routeParams'));
const params = count ? { count } : {};
await initToken();
const userInfo = await getUser(id);
let data = await (exclude_replies ? getUserTweets(id, params) : getUserTweetsAndReplies(id, params));
if (!include_rts) {
data = excludeRetweet(data);
}
const profileImageUrl = userInfo.profile_image_url || userInfo.profile_image_url_https;
return {
title: `Twitter @${userInfo.name}`,
link: `https://twitter.com/${userInfo.screen_name}`,
image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
description: userInfo.description,
item: utils.ProcessFeed(ctx, {
data,
}),
};
};

23
lib/routes/twitter/tweet.ts
View File

@ -1,5 +1,7 @@
import { Route } from '@/types';
import webApiImpl from './mobile-api/tweet';
import api from './api';
import utils from './utils';
import { fallback, queryToBoolean } from '@/utils/readable-social';
export const route: Route = {
path: '/tweet/:id/status/:status/:original?',
@ -33,5 +35,22 @@ export const route: Route = {
};
async function handler(ctx) {
return await webApiImpl(ctx);
const id = ctx.req.param('id');
const status = ctx.req.param('status');
const routeParams = new URLSearchParams(ctx.req.param('original'));
const original = fallback(undefined, queryToBoolean(routeParams.get('original')), false);
const params = { focalTweetId: status };
await api.init();
const userInfo = await api.getUser(id);
const data = await api.getUserTweet(id, params);
const profileImageUrl = userInfo.profile_image_url || userInfo.profile_image_url_https;
const item = original && config.isPackage ? data : utils.ProcessFeed(ctx, { data });
return {
title: `Twitter @${userInfo.name}`,
link: `https://twitter.com/${userInfo.screen_name}/status/${status}`,
image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
description: userInfo.description,
item,
};
}

28
lib/routes/twitter/user.ts
View File

@ -1,5 +1,6 @@
import { Route } from '@/types';
import webApiImpl from './mobile-api/user';
import utils from './utils';
import api from './api';
export const route: Route = {
path: '/user/:id/:routeParams?',
@ -39,5 +40,28 @@ export const route: Route = {
};
async function handler(ctx) {
return await webApiImpl(ctx);
const id = ctx.req.param('id');
// For compatibility
const { count, exclude_replies, include_rts } = utils.parseRouteParams(ctx.req.param('routeParams'));
const params = count ? { count } : {};
await api.init();
const userInfo = await api.getUser(id);
let data = await (exclude_replies ? api.getUserTweets(id, params) : api.getUserTweetsAndReplies(id, params));
if (!include_rts) {
data = utils.excludeRetweet(data);
}
const profileImageUrl = userInfo?.profile_image_url || userInfo?.profile_image_url_https;
return {
title: `Twitter @${userInfo?.name}`,
link: `https://twitter.com/${userInfo?.screen_name}`,
image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
description: userInfo?.description,
item: utils.ProcessFeed(ctx, {
data,
}),
};
}

13
lib/routes/twitter/utils.ts
View File

@ -474,4 +474,15 @@ const parseRouteParams = (routeParams) => {
return { count, exclude_replies, include_rts, force_web_api };
};
export default { ProcessFeed, getAppClient, parseRouteParams };
export const excludeRetweet = function (tweets) {
const excluded = [];
for (const t of tweets) {
if (t.retweeted_status) {
continue;
}
excluded.push(t);
}
return excluded;
};
export default { ProcessFeed, getAppClient, parseRouteParams, excludeRetweet };