Skip to content

Commit

Permalink
Clean up code
Browse files Browse the repository at this point in the history
  • Loading branch information
@dslucas
dslucas committed Nov 28, 2022
1 parent d832b00 commit 93e3160
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 108 deletions.
6 changes: 3 additions & 3 deletions src/app/social-media-item.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,9 @@ export class SocialMediaItemService {
return from(
this.twitterApiService.getTweets({
startDateTimeMs,
// Subtract 1 minute from the end time because the Twitter API
// sometimes returns an error if we request data for the most recent
// minute of time.
// Subtract 1 minute from the end time because the Twitter API sometimes
// returns an error if we request data for the most recent minute of
// time.
endDateTimeMs: endDateTimeMs - 60000,
})
).pipe(map((response: GetTweetsResponse) => response.tweets));
Expand Down
32 changes: 19 additions & 13 deletions src/common-types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ export interface TwitterApiResponse {
results: TweetObject[];
}

// Tweet object format returned by the Enterprise Twitter API.
//
// From twitter documentation: When ingesting Tweet data the main object is the
// Tweet Object, which is a parent object to several child objects. For
// example, all Tweets include a User object that describes who authored the
Expand Down Expand Up @@ -167,19 +169,31 @@ export interface TweetObject {
source?: string;
}

// Tweet object format returned by the v2 Twitter API.
//
// See
// https://developer.twitter.com/en/docs/twitter-api/data-dictionary/introduction
// for more details.
export interface V2TweetObject {
attachments: V2Attachments;
attachments?: V2Attachments;
author_id: string;
created_at: string;
entities: V2Entities;
entities?: V2Entities;
id: string;
lang: string;
public_metrics: V2PublicMetrics;
referenced_tweets: V2ReferencedTweet[];
source: string;
referenced_tweets?: V2ReferencedTweet[];
source?: string;
text: string;
}

interface V2Entities {
hashtags?: V2Hashtags[];
mentions?: V2Mentions[];
referenced_tweets?: V2ReferencedTweet[];
urls?: V2Url[];
}

interface V2PublicMetrics {
like_count: number;
quote_count: number;
Expand All @@ -192,21 +206,14 @@ interface V2ReferencedTweet {
type: string;
}

interface V2Entities {
hashtags?: V2Hashtags[];
mentions?: V2Mentions[];
referenced_tweets?: V2ReferencedTweet[];
urls?: V2Url[];
}

interface V2Mentions {
start: number;
end: number;
username: string;
id: string;
}

interface V2Hashtags {
export interface V2Hashtags {
start: number;
end: number;
tag: string;
Expand Down Expand Up @@ -241,7 +248,6 @@ interface V2Attachments {
media_keys: string[];
}

// TODO: Maybe remove all unused fields?
export interface TwitterUser {
id_str: string;
screen_name: string;
Expand Down
201 changes: 109 additions & 92 deletions src/server/middleware/twitter.middleware.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import {
MuteTwitterUsersRequest,
MuteTwitterUsersResponse,
Tweet,
TweetEntities,
TweetHashtag,
TweetMedia,
TweetObject,
Expand Down Expand Up @@ -60,29 +61,19 @@ export async function getTweets(
) {
let twitterDataPromise: Promise<TwitterApiResponse>;

// if (fs.existsSync('src/server/twitter_sample_results.json')) {
// twitterDataPromise = loadLocalTwitterData();
// } else if (enterpriseSearchCredentialsAreValid(apiCredentials)) {
// twitterDataPromise = loadTwitterData(apiCredentials, req.body);
// } else if (v2SearchCredentialsAreValid(apiCredentials)) {
// twitterDataPromise = loadTwitterDataV2(apiCredentials, req.body);
// } else {
// res.send(new Error('Invalid Search API credentials'));
// return;
// }

// const enterpriseResults = (
// await loadTwitterData(apiCredentials, req.body)
// ).results.map(parseTweet);
// const v2Results = (
// await loadTwitterDataV2(apiCredentials, req.body)
// ).results.map(parseTweet);

// console.log(JSON.stringify(enterpriseResults));
// console.log(JSON.stringify(v2Results));
if (fs.existsSync('src/server/twitter_sample_results.json')) {
twitterDataPromise = loadLocalTwitterData();
} else if (v2SearchCredentialsAreValid(apiCredentials)) {
twitterDataPromise = loadTwitterDataV2(apiCredentials, req.body);
} else if (enterpriseSearchCredentialsAreValid(apiCredentials)) {
twitterDataPromise = loadTwitterData(apiCredentials, req.body);
} else {
res.send(new Error('No valid Twitter API credentials'));
return;
}

try {
const twitterData = await loadTwitterData(apiCredentials, req.body);
const twitterData = await twitterDataPromise;
const tweets = twitterData.results.map(parseTweet);
res.send({ tweets, nextPageToken: twitterData.next } as GetTweetsResponse);
} catch (e) {
Expand Down Expand Up @@ -331,83 +322,48 @@ function loadTwitterDataV2(
if (!user) {
throw new Error('No user credentials in GetTweetsRequest');
}

const params = {
// Include next_token if it's part of the request.
...(request.nextPageToken && { next_token: request.nextPageToken }),
...{
query: `(@${user} OR url:twitter.com/${user}) -from:${user} -is:retweet`,
max_results: BATCH_SIZE,
'user.fields': 'id,name,username,profile_image_url,verified',
expansions: 'author_id,attachments.media_keys,referenced_tweets.id',
start_time: formatTimestampForV2(request.startDateTimeMs),
end_time: formatTimestampForV2(request.endDateTimeMs),
'media.fields': 'url,type',
'tweet.fields':
'attachments,created_at,id,entities,lang,public_metrics,source,text',
},
};

return axios
.get<TwitterApiResponse>(requestUrl, {
headers: {
authorization: `Bearer ${credentials.bearerToken}`,
},
params: {
query: `(@${user} OR url:twitter.com/${user}) -from:${user} -is:retweet`,
max_results: BATCH_SIZE,
'user.fields': 'id,name,username,profile_image_url,verified',
expansions: 'author_id,attachments.media_keys,referenced_tweets.id',
start_time: formatTimestampForV2(request.startDateTimeMs),
end_time: formatTimestampForV2(request.endDateTimeMs),
'media.fields': 'url,type',
'tweet.fields':
'attachments,created_at,id,entities,lang,public_metrics,source,text',
},
params,
transformResponse: [
(data, _) => {
const response = JSON.parse(data);
const tweets: V2TweetObject[] = response.data ?? [];
const includes: V2Includes = response.includes ?? {};
const toReturn = {
const parsed = JSON.parse(data);
const tweets: V2TweetObject[] = parsed.data ?? [];
const includes: V2Includes = parsed.includes ?? {};
return <TwitterApiResponse>{
results: tweets.map((tweet) => packV2Tweet(tweet, includes)),
next: response.meta.next_token ?? '',
next: parsed.meta.next_token,
};
return toReturn;
},
],
})
.then((response) => response.data);
}

// Packs a Tweet object response from the V2 Search PI into the Enterprise
// Search API response object format.
// Packs a Tweet response object from the v2 Search API format into the
// Enterprise Search API format.
function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject {
const entities = {
hashtags: tweet.entities?.hashtags?.map(
(hashtag) =>
<TweetHashtag>{
indices: [hashtag.start, hashtag.end],
text: hashtag.tag,
}
),
// TODO: Handle this
symbols: [],
urls: tweet.entities?.urls?.map(
(url) =>
<TweetUrl>{
display_url: url.display_url,
expanded_url: url.extended_url,
indices: [url.start, url.end],
}
),
user_mentions: tweet.entities.mentions?.map(
(mention) =>
<TweetUserMention>{
id_str: mention.id,
indices: [mention.start, mention.end],
screen_name: mention.username,
}
),
media: tweet.attachments?.media_keys
?.flatMap((media_key) => {
const media = includes.media?.find(
(media) => media.media_key === media_key
);
if (!media) {
throw new Error('Unable to find media');
}
return <TweetMedia>{
indices: [0, 0], // Indices not available in V2.
media_url: media.url ?? '',
type: media.type,
};
})
.filter((media) => media.type === 'photo'),
};
const entities = packEntities(tweet, includes);

const tweetObject: TweetObject = {
created_at: tweet.created_at,
Expand All @@ -417,10 +373,9 @@ function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject {
// entities field is.
entities: entities,
extended_entities: entities,
// No display_text_range, truncated, or extended_tweet fields because the
// V2 API does not truncate the text.
// `display_text_range` omitted because v2 does not truncate the text.
favorite_count: tweet.public_metrics.like_count,
// favorited is not available in V2 API.
// `favorited` omitted because it is not available in v2..
in_reply_to_status_id: tweet.entities?.referenced_tweets?.find(
(tweet) => tweet.type === 'replied_to'
)?.id,
Expand All @@ -433,8 +388,8 @@ function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject {
};

if (tweetObject.truncated) {
// The Enteprise API populates the extended_tweet field if the tweet is
// truncated.
// Enteprise populates the extended_tweet field if the tweet is truncated,
// so we add it manually here for consistency.
tweetObject.extended_tweet = {
full_text: tweetObject.text,
display_text_range: [0, 0], // Indices not available in v2.
Expand All @@ -445,7 +400,72 @@ function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject {
return tweetObject;
}

function packEntities(
tweet: V2TweetObject,
includes: V2Includes
): TweetEntities {
const entities: TweetEntities = {};

if (tweet.entities && tweet.entities.hashtags) {
entities.hashtags = tweet.entities.hashtags.map(
(hashtag) =>
<TweetHashtag>{
indices: [hashtag.start, hashtag.end],
text: hashtag.tag,
}
);
}

if (tweet.entities && tweet.entities.urls) {
entities.urls = tweet.entities.urls.map(
(url) =>
<TweetUrl>{
display_url: url.display_url,
expanded_url: url.extended_url,
indices: [url.start, url.end],
}
);
}

if (tweet.entities && tweet.entities.mentions) {
entities.user_mentions = tweet.entities.mentions.map(
(mention) =>
<TweetUserMention>{
id_str: mention.id,
indices: [mention.start, mention.end],
screen_name: mention.username,
}
);
}

if (tweet.attachments && tweet.attachments.media_keys) {
entities.media = tweet.attachments.media_keys
.flatMap((media_key) => {
// v2 includes the media fields (like media_url) in a separate
// `includes` object, so we search there for the media item in question.
const media = includes.media?.find(
(media) => media.media_key === media_key
);
// This shouldn't happen, but we throw an error if it does.
if (!media) {
throw new Error('Unable to find media');
}
return <TweetMedia>{
indices: [0, 0], // Indices not available in v2.
media_url: media.url,
type: media.type,
};
})
// Filter for photos because that's all we display.
.filter((media) => media.type === 'photo');
}

return entities;
}

function getUser(id: string, includes: V2Includes): TwitterUser {
// v2 includes the user fields (like profile_image_url) in a separate
// `includes` object, so we search there for the media item in question.
const user = includes.users?.find((user) => user.id === id);
if (!user) {
throw new Error('Unable to find user');
Expand Down Expand Up @@ -564,9 +584,6 @@ function parseTweet(tweetObject: TweetObject): Tweet {
if (tweetObject.extended_entities && tweetObject.extended_entities.media) {
tweet.hasImage = true;
}
// console.log(tweet.text);
// console.log(tweet.text.length);
// console.log(tweet.truncated);
return tweet;
}

Expand Down Expand Up @@ -595,8 +612,8 @@ function formatTimestamp(ms: number): string {
);
}

// Format a millisecond-based timestamp into the YYYY-MM-DDTHH:mm:ssZ date format
// suitable for the v2 Twitter API, as defined in:
// Format a millisecond-based timestamp into the YYYY-MM-DDTHH:mm:ssZ date
// format suitable for the v2 Twitter API, as defined in:
// https://developer.twitter.com/en/docs/twitter-api/tweets/search/api-reference/get-tweets-search-all
function formatTimestampForV2(ms: number): string {
const date = new Date(ms);
Expand Down

0 comments on commit 93e3160

Please sign in to comment.