283 lines
10 KiB
JavaScript
283 lines
10 KiB
JavaScript
const { baseUrl, gqlMap, gqlFeatures, consumerKey, consumerSecret } = require('./constants');
|
|
const config = require('@/config').value;
|
|
const logger = require('@/utils/logger');
|
|
const got = require('@/utils/got');
|
|
const OAuth = require('oauth-1.0a');
|
|
const CryptoJS = require('crypto-js');
|
|
const queryString = require('query-string');
|
|
|
|
let tokenIndex = 0;
|
|
|
|
const twitterGot = async (url, params) => {
|
|
if (!config.twitter.oauthTokens?.length || !config.twitter.oauthTokenSecrets?.length || config.twitter.oauthTokens.length !== config.twitter.oauthTokenSecrets.length) {
|
|
throw Error('Invalid twitter oauth tokens');
|
|
}
|
|
|
|
const oauth = OAuth({
|
|
consumer: {
|
|
key: consumerKey,
|
|
secret: consumerSecret,
|
|
},
|
|
signature_method: 'HMAC-SHA1',
|
|
hash_function: (base_string, key) => CryptoJS.HmacSHA1(base_string, key).toString(CryptoJS.enc.Base64),
|
|
});
|
|
|
|
const token = {
|
|
key: config.twitter.oauthTokens[tokenIndex],
|
|
secret: config.twitter.oauthTokenSecrets[tokenIndex],
|
|
};
|
|
tokenIndex++;
|
|
if (tokenIndex >= config.twitter.oauthTokens.length) {
|
|
tokenIndex = 0;
|
|
}
|
|
|
|
const requestData = {
|
|
url: `${url}?${queryString.stringify(params)}`,
|
|
method: 'GET',
|
|
headers: {
|
|
connection: 'keep-alive',
|
|
'content-type': 'application/json',
|
|
'x-twitter-active-user': 'yes',
|
|
authority: 'api.twitter.com',
|
|
'accept-encoding': 'gzip',
|
|
'accept-language': 'en-US,en;q=0.9',
|
|
accept: '*/*',
|
|
DNT: '1',
|
|
},
|
|
};
|
|
|
|
const response = await got(requestData.url, {
|
|
headers: oauth.toHeader(oauth.authorize(requestData, token)),
|
|
});
|
|
|
|
return response.data;
|
|
};
|
|
|
|
const paginationTweets = async (endpoint, userId, variables, path) => {
|
|
const { data } = await twitterGot(baseUrl + endpoint, {
|
|
variables: JSON.stringify({
|
|
...variables,
|
|
rest_id: userId,
|
|
}),
|
|
features: gqlFeatures,
|
|
});
|
|
|
|
let instructions;
|
|
if (!path) {
|
|
instructions = data.user_result.result.timeline_response.timeline.instructions;
|
|
} else {
|
|
instructions = data;
|
|
path.forEach((p) => (instructions = instructions[p]));
|
|
instructions = instructions.instructions;
|
|
}
|
|
|
|
return instructions.filter((i) => i.__typename === 'TimelineAddEntries' || i.type === 'TimelineAddEntries')[0].entries;
|
|
};
|
|
|
|
const timelineTweets = (userId, params = {}) =>
|
|
paginationTweets(gqlMap.UserWithProfileTweets, userId, {
|
|
...params,
|
|
withQuickPromoteEligibilityTweetFields: true,
|
|
});
|
|
|
|
const timelineTweetsAndReplies = (userId, params = {}) =>
|
|
paginationTweets(gqlMap.UserWithProfileTweetsAndReplies, userId, {
|
|
...params,
|
|
count: 20,
|
|
});
|
|
|
|
const timelineMedia = (userId, params = {}) => paginationTweets(gqlMap.MediaTimeline, userId, params);
|
|
|
|
// const timelineLikes = (userId, params = {}) => paginationTweets(gqlMap.Likes, userId, params);
|
|
|
|
const timelineKeywords = (keywords, params = {}) =>
|
|
paginationTweets(
|
|
gqlMap.SearchTimeline,
|
|
null,
|
|
{
|
|
...params,
|
|
rawQuery: keywords,
|
|
count: 20,
|
|
product: 'Latest',
|
|
withDownvotePerspective: false,
|
|
withReactionsMetadata: false,
|
|
withReactionsPerspective: false,
|
|
},
|
|
['search_by_raw_query', 'search_timeline', 'timeline']
|
|
);
|
|
|
|
const tweetDetail = (userId, params) =>
|
|
paginationTweets(
|
|
gqlMap.TweetDetail,
|
|
userId,
|
|
{
|
|
...params,
|
|
includeHasBirdwatchNotes: false,
|
|
includePromotedContent: false,
|
|
withBirdwatchNotes: false,
|
|
withVoice: false,
|
|
withV2Timeline: true,
|
|
},
|
|
['threaded_conversation_with_injections_v2']
|
|
);
|
|
|
|
function gatherLegacyFromData(entries, filterNested = undefined, userId = undefined) {
|
|
const tweets = [];
|
|
const filteredEntries = [];
|
|
entries.forEach((entry) => {
|
|
const entryId = entry.entryId;
|
|
if (entryId) {
|
|
if (entryId.startsWith('tweet-')) {
|
|
filteredEntries.push(entry);
|
|
}
|
|
if (filterNested && filterNested.some((f) => entryId.startsWith(f))) {
|
|
filteredEntries.push(...entry.content.items);
|
|
}
|
|
}
|
|
});
|
|
filteredEntries.forEach((entry) => {
|
|
if (entry.entryId) {
|
|
const content = entry.content || entry.item;
|
|
let tweet = content?.content?.tweetResult?.result || content?.itemContent?.tweet_results?.result;
|
|
if (tweet && tweet.tweet) {
|
|
tweet = tweet.tweet;
|
|
}
|
|
if (tweet) {
|
|
const retweet = tweet.legacy?.retweeted_status_result?.result;
|
|
for (const t of [tweet, retweet]) {
|
|
if (!t?.legacy) {
|
|
continue;
|
|
}
|
|
t.legacy.user = t.core?.user_result?.result?.legacy || t.core?.user_results?.result?.legacy;
|
|
const quote = t.quoted_status_result?.result;
|
|
if (quote) {
|
|
t.legacy.quoted_status = quote.legacy;
|
|
t.legacy.quoted_status.user = quote.core.user_result?.result?.legacy || quote.core.user_results?.result?.legacy;
|
|
}
|
|
}
|
|
const legacy = tweet.legacy;
|
|
if (legacy) {
|
|
if (retweet) {
|
|
legacy.retweeted_status = retweet.legacy;
|
|
}
|
|
if (userId === undefined || legacy.user_id_str === userId + '') {
|
|
tweets.push(legacy);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
return tweets;
|
|
}
|
|
|
|
const getUserTweetsByID = async (id, params = {}) => gatherLegacyFromData(await timelineTweets(id, params));
|
|
// TODO: show the whole conversation instead of just the reply tweet
|
|
const getUserTweetsAndRepliesByID = async (id, params = {}) => gatherLegacyFromData(await timelineTweetsAndReplies(id, params), ['profile-conversation-'], id);
|
|
const getUserMediaByID = async (id, params = {}) => gatherLegacyFromData(await timelineMedia(id, params));
|
|
// const getUserLikesByID = async (id, params = {}) => gatherLegacyFromData(await timelineLikes(id, params));
|
|
const getUserTweetByStatus = async (id, params = {}) => gatherLegacyFromData(await tweetDetail(id, params), ['homeConversation-', 'conversationthread-']);
|
|
|
|
const excludeRetweet = function (tweets) {
|
|
const excluded = [];
|
|
for (const t of tweets) {
|
|
if (t.retweeted_status) {
|
|
continue;
|
|
}
|
|
excluded.push(t);
|
|
}
|
|
return excluded;
|
|
};
|
|
|
|
const userByScreenName = (screenName) =>
|
|
twitterGot(`${baseUrl}${gqlMap.UserResultByScreenName}`, {
|
|
variables: `{"screen_name":"${screenName}","withHighlightedLabel":true}`,
|
|
features: gqlFeatures,
|
|
});
|
|
const userByRestId = (restId) =>
|
|
twitterGot(`${baseUrl}${gqlMap.UserByRestId}`, {
|
|
variables: `{"userId":"${restId}","withHighlightedLabel":true}`,
|
|
features: gqlFeatures,
|
|
});
|
|
const userByAuto = (id) => {
|
|
if (id.startsWith('+')) {
|
|
return userByRestId(id.slice(1));
|
|
}
|
|
return userByScreenName(id);
|
|
};
|
|
const getUserData = (cache, id) => cache.tryGet(`twitter-userdata-${id}`, () => userByAuto(id));
|
|
const getUserID = async (cache, id) => {
|
|
const userData = await getUserData(cache, id);
|
|
return (userData.data?.user || userData.data?.user_result)?.result?.rest_id;
|
|
};
|
|
const getUser = async (cache, id) => {
|
|
const userData = await getUserData(cache, id);
|
|
return (userData.data?.user || userData.data?.user_result)?.result?.legacy;
|
|
};
|
|
|
|
const cacheTryGet = async (cache, _id, params, func) => {
|
|
const id = await getUserID(cache, _id);
|
|
if (id === undefined) {
|
|
throw Error('User not found');
|
|
}
|
|
const funcName = func.name;
|
|
const paramsString = JSON.stringify(params);
|
|
return cache.tryGet(`twitter:${id}:${funcName}:${paramsString}`, () => func(id, params), config.cache.routeExpire, false);
|
|
};
|
|
|
|
// returns:
|
|
// 1. nothing for some users
|
|
// 2. HOT tweets for the other users, instead of the LATEST ones
|
|
const _getUserTweets = (cache, id, params = {}) => cacheTryGet(cache, id, params, getUserTweetsByID);
|
|
// workaround for the above issue:
|
|
// 1. getUserTweetsAndReplies return LATEST tweets and replies, which requires filtering
|
|
// a. if one replies a lot (e.g. elonmusk), there is sometimes no tweets left after filtering, caching may help
|
|
// 2. getUserMedia return LATEST media tweets, which is a good plus
|
|
const getUserTweets = async (cache, id, params = {}) => {
|
|
let tweets = [];
|
|
const rest_id = await getUserID(cache, id);
|
|
await Promise.all(
|
|
[_getUserTweets, getUserTweetsAndReplies, getUserMedia].map(async (func) => {
|
|
try {
|
|
tweets.push(...(await func(cache, id, params)));
|
|
} catch (e) {
|
|
logger.warn(`Failed to get tweets for ${id} with ${func.name}: ${e}`);
|
|
}
|
|
})
|
|
);
|
|
|
|
const cacheKey = `twitter:user:tweets-cache:${rest_id}`;
|
|
let cacheValue = await cache.get(cacheKey);
|
|
if (cacheValue) {
|
|
cacheValue = JSON.parse(cacheValue);
|
|
if (cacheValue && cacheValue.length) {
|
|
tweets = cacheValue.concat(tweets);
|
|
}
|
|
}
|
|
const idSet = new Set();
|
|
tweets = tweets
|
|
.filter((tweet) => !tweet.in_reply_to_screen_name) // exclude replies
|
|
.map((tweet) => (idSet.has(tweet.id_str) ? null : idSet.add(tweet.id_str)) && tweet) // deduplicate
|
|
.filter(Boolean) // remove null
|
|
.sort((a, b) => b.id_str - a.id_str) // desc
|
|
.slice(0, 20);
|
|
cache.set(cacheKey, JSON.stringify(tweets));
|
|
return tweets;
|
|
};
|
|
const getUserTweetsAndReplies = (cache, id, params = {}) => cacheTryGet(cache, id, params, getUserTweetsAndRepliesByID);
|
|
const getUserMedia = (cache, id, params = {}) => cacheTryGet(cache, id, params, getUserMediaByID);
|
|
// const getUserLikes = (cache, id, params = {}) => cacheTryGet(cache, id, params, getUserLikesByID);
|
|
const getUserTweet = (cache, id, params) => cacheTryGet(cache, id, params, getUserTweetByStatus);
|
|
|
|
const getSearch = async (keywords, params = {}) => gatherLegacyFromData(await timelineKeywords(keywords, params));
|
|
|
|
module.exports = {
|
|
getUser,
|
|
getUserTweets,
|
|
getUserTweetsAndReplies,
|
|
getUserMedia,
|
|
// getUserLikes,
|
|
excludeRetweet,
|
|
getSearch,
|
|
getUserTweet,
|
|
};
|