From b66339f7b6b8bbad9afd2c0c32785f5d8150c929 Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Wed, 23 Nov 2022 17:07:25 -0500 Subject: [PATCH 1/9] Initial working attempt at supporting the v2 Search API --- src/app/social-media-item.service.ts | 23 +- src/common-types.ts | 92 ++++++-- src/server/middleware/twitter.middleware.ts | 221 ++++++++++++++++++-- src/server/serving.ts | 1 + 4 files changed, 290 insertions(+), 47 deletions(-) diff --git a/src/app/social-media-item.service.ts b/src/app/social-media-item.service.ts index 1150056..d9c1dba 100644 --- a/src/app/social-media-item.service.ts +++ b/src/app/social-media-item.service.ts @@ -126,11 +126,11 @@ export class SocialMediaItemService { ): Observable { return from( this.twitterApiService.getTweets({ - fromDate: formatTimestamp(startDateTimeMs), + startDateTimeMs, // Subtract 1 minute from the end time because the Twitter API // sometimes returns an error if we request data for the most recent // minute of time. - toDate: formatTimestamp(endDateTimeMs - 60000), + endDateTimeMs: endDateTimeMs - 60000, }) ).pipe(map((response: GetTweetsResponse) => response.tweets)); } @@ -166,22 +166,3 @@ export class SocialMediaItemService { .pipe(map(scores => ({ item, scores }))); } } - -// Format a millisecond-based timestamp into a date format suitable for the -// Twitter API, as defined in: -// https://developer.twitter.com/en/docs/tweets/search/api-reference/enterprise-search -function formatTimestamp(ms: number): string { - const date = new Date(ms); - const MM = date.getUTCMonth() + 1; // getMonth() is zero-based - const dd = date.getUTCDate(); - const hh = date.getUTCHours(); - const mm = date.getUTCMinutes(); - - return ( - `${date.getFullYear()}` + - `${(MM > 9 ? '' : '0') + MM}` + - `${(dd > 9 ? '' : '0') + dd}` + - `${(hh > 9 ? '' : '0') + hh}` + - `${(mm > 9 ? '' : '0') + mm}` - ); -} diff --git a/src/common-types.ts b/src/common-types.ts index 4128094..aaa2b86 100644 --- a/src/common-types.ts +++ b/src/common-types.ts @@ -79,8 +79,8 @@ export interface CreatePdfResponse { export interface GetTweetsRequest { credentials?: firebase.auth.UserCredential; nextPageToken?: string; - fromDate: string; // yyyymmddhhmm format is expected here. - toDate: string; // yyyymmddhhmm format is expected here. + startDateTimeMs: number; + endDateTimeMs: number; } export interface GetTweetsResponse { @@ -125,16 +125,9 @@ export interface HideRepliesTwitterResponse { export interface TwitterApiResponse { next: string; - requestParameters: TwitterApiRequestParams; results: TweetObject[]; } -interface TwitterApiRequestParams { - fromDate: string; - toDate: string; - maxResults: number; -} - // From twitter documentation: When ingesting Tweet data the main object is the // Tweet Object, which is a parent object to several child objects. For // example, all Tweets include a User object that describes who authored the @@ -174,6 +167,81 @@ export interface TweetObject { source?: string; } +export interface V2TweetObject { + attachments: V2Attachments; + author_id: string; + created_at: string; + entities: V2Entities; + id: string; + lang: string; + public_metrics: V2PublicMetrics; + referenced_tweets: V2ReferencedTweet[]; + source: string; + text: string; +} + +interface V2PublicMetrics { + like_count: number; + quote_count: number; + reply_count: number; + retweet_count: number; +} + +interface V2ReferencedTweet { + id: string; + type: string; +} + +interface V2Entities { + hashtags?: V2Hashtags[]; + mentions?: V2Mentions[]; + referenced_tweets?: V2ReferencedTweet[]; + urls?: V2Url[]; +} + +interface V2Mentions { + start: number; + end: number; + username: string; + id: string; +} + +interface V2Hashtags { + start: number; + end: number; + tag: string; +} + +interface V2Url { + display_url: string; + extended_url: string; + start: number; + end: number; +} + +export interface V2Includes { + media?: V2Media[]; + users?: V2Users[]; +} + +interface V2Media { + media_key: string; + type: string; + url: string; +} + +interface V2Users { + profile_image_url: string; + name: string; + username: string; + verified: boolean; + id: string; +} +interface V2Attachments { + media_keys: string[]; +} + +// TODO: Maybe remove all unused fields? export interface TwitterUser { id_str: string; screen_name: string; @@ -206,7 +274,7 @@ interface Symbols { text: string; } -interface TweetUserMention { +export interface TweetUserMention { id?: number; id_str?: string; indices: Indices; @@ -214,7 +282,7 @@ interface TweetUserMention { screen_name: string; } -interface TweetMedia { +export interface TweetMedia { id_str?: string; media_url: string; type: string; @@ -234,7 +302,7 @@ interface TweetMediaDimensions { resize: string; } -interface TweetUrl { +export interface TweetUrl { display_url?: string; expanded_url?: string; indices: Indices; diff --git a/src/server/middleware/twitter.middleware.ts b/src/server/middleware/twitter.middleware.ts index a702240..b57115b 100644 --- a/src/server/middleware/twitter.middleware.ts +++ b/src/server/middleware/twitter.middleware.ts @@ -30,8 +30,15 @@ import { MuteTwitterUsersRequest, MuteTwitterUsersResponse, Tweet, + TweetHashtag, + TweetMedia, TweetObject, + TweetUrl, + TweetUserMention, TwitterApiResponse, + TwitterUser, + V2Includes, + V2TweetObject, } from '../../common-types'; import { TwitterApiCredentials } from '../serving'; @@ -53,18 +60,29 @@ export async function getTweets( ) { let twitterDataPromise: Promise; - if (fs.existsSync('src/server/twitter_sample_results.json')) { - twitterDataPromise = loadLocalTwitterData(); - } else { - if (!enterpriseSearchCredentialsAreValid(apiCredentials)) { - res.send(new Error('Invalid Twitter Enterprise Search API credentials')); - return; - } - twitterDataPromise = loadTwitterData(apiCredentials, req.body); - } + // if (fs.existsSync('src/server/twitter_sample_results.json')) { + // twitterDataPromise = loadLocalTwitterData(); + // } else if (enterpriseSearchCredentialsAreValid(apiCredentials)) { + // twitterDataPromise = loadTwitterData(apiCredentials, req.body); + // } else if (v2SearchCredentialsAreValid(apiCredentials)) { + // twitterDataPromise = loadTwitterDataV2(apiCredentials, req.body); + // } else { + // res.send(new Error('Invalid Search API credentials')); + // return; + // } + + // const enterpriseResults = ( + // await loadTwitterData(apiCredentials, req.body) + // ).results.map(parseTweet); + // const v2Results = ( + // await loadTwitterDataV2(apiCredentials, req.body) + // ).results.map(parseTweet); + + // console.log(JSON.stringify(enterpriseResults)); + // console.log(JSON.stringify(v2Results)); try { - const twitterData = await twitterDataPromise; + const twitterData = await loadTwitterData(apiCredentials, req.body); const tweets = twitterData.results.map(parseTweet); res.send({ tweets, nextPageToken: twitterData.next } as GetTweetsResponse); } catch (e) { @@ -278,11 +296,11 @@ function loadTwitterData( maxResults: BATCH_SIZE, }; - if (request.fromDate) { - twitterApiRequest.fromDate = request.fromDate; + if (request.startDateTimeMs) { + twitterApiRequest.fromDate = formatTimestamp(request.startDateTimeMs); } - if (request.toDate) { - twitterApiRequest.toDate = request.toDate; + if (request.endDateTimeMs) { + twitterApiRequest.toDate = formatTimestamp(request.endDateTimeMs); } if (request.nextPageToken) { twitterApiRequest.next = request.nextPageToken; @@ -304,6 +322,143 @@ function loadTwitterData( }); } +function loadTwitterDataV2( + credentials: TwitterApiCredentials, + request: GetTweetsRequest +): Promise { + const requestUrl = 'https://api.twitter.com/2/tweets/search/all'; + const user = request.credentials?.additionalUserInfo?.username; + if (!user) { + throw new Error('No user credentials in GetTweetsRequest'); + } + return axios + .get(requestUrl, { + headers: { + authorization: `Bearer ${credentials.bearerToken}`, + }, + params: { + query: `(@${user} OR url:twitter.com/${user}) -from:${user} -is:retweet`, + max_results: BATCH_SIZE, + 'user.fields': 'id,name,username,profile_image_url,verified', + expansions: 'author_id,attachments.media_keys,referenced_tweets.id', + start_time: formatTimestampForV2(request.startDateTimeMs), + end_time: formatTimestampForV2(request.endDateTimeMs), + 'media.fields': 'url,type', + 'tweet.fields': + 'attachments,created_at,id,entities,lang,public_metrics,source,text', + }, + transformResponse: [ + (data, _) => { + const response = JSON.parse(data); + const tweets: V2TweetObject[] = response.data ?? []; + const includes: V2Includes = response.includes ?? {}; + const toReturn = { + results: tweets.map((tweet) => packV2Tweet(tweet, includes)), + next: response.meta.next_token ?? '', + }; + return toReturn; + }, + ], + }) + .then((response) => response.data); +} + +// Packs a Tweet object response from the V2 Search PI into the Enterprise +// Search API response object format. +function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { + const entities = { + hashtags: tweet.entities?.hashtags?.map( + (hashtag) => + { + indices: [hashtag.start, hashtag.end], + text: hashtag.tag, + } + ), + // TODO: Handle this + symbols: [], + urls: tweet.entities?.urls?.map( + (url) => + { + display_url: url.display_url, + expanded_url: url.extended_url, + indices: [url.start, url.end], + } + ), + user_mentions: tweet.entities.mentions?.map( + (mention) => + { + id_str: mention.id, + indices: [mention.start, mention.end], + screen_name: mention.username, + } + ), + media: tweet.attachments?.media_keys + ?.flatMap((media_key) => { + const media = includes.media?.find( + (media) => media.media_key === media_key + ); + if (!media) { + throw new Error('Unable to find media'); + } + return { + indices: [0, 0], // Indices not available in V2. + media_url: media.url ?? '', + type: media.type, + }; + }) + .filter((media) => media.type === 'photo'), + }; + + const tweetObject: TweetObject = { + created_at: tweet.created_at, + id_str: tweet.id, + text: tweet.text, + // The Enterprise API entities field is not always complete, but the v2 + // entities field is. + entities: entities, + extended_entities: entities, + // No display_text_range, truncated, or extended_tweet fields because the + // V2 API does not truncate the text. + favorite_count: tweet.public_metrics.like_count, + // favorited is not available in V2 API. + in_reply_to_status_id: tweet.entities?.referenced_tweets?.find( + (tweet) => tweet.type === 'replied_to' + )?.id, + lang: tweet.lang, + reply_count: tweet.public_metrics.reply_count, + retweet_count: tweet.public_metrics.retweet_count, + source: tweet.source, + truncated: tweet.text.length > 140, + user: getUser(tweet.author_id, includes), + }; + + if (tweetObject.truncated) { + // The Enteprise API populates the extended_tweet field if the tweet is + // truncated. + tweetObject.extended_tweet = { + full_text: tweetObject.text, + display_text_range: [0, 0], // Indices not available in v2. + entities, + }; + } + + return tweetObject; +} + +function getUser(id: string, includes: V2Includes): TwitterUser { + const user = includes.users?.find((user) => user.id === id); + if (!user) { + throw new Error('Unable to find user'); + } + return { + id_str: user?.id, + profile_image_url: user?.profile_image_url, + name: user?.name, + screen_name: user?.username, + verified: user?.verified, + }; +} + function loadLocalTwitterData(): Promise { return fs.promises .readFile('src/server/twitter_sample_results.json') @@ -326,6 +481,12 @@ function enterpriseSearchCredentialsAreValid( ); } +function v2SearchCredentialsAreValid( + credentials: TwitterApiCredentials +): boolean { + return !!credentials.bearerToken; +} + function standardApiCredentialsAreValid( credentials: TwitterApiCredentials ): boolean { @@ -403,6 +564,9 @@ function parseTweet(tweetObject: TweetObject): Tweet { if (tweetObject.extended_entities && tweetObject.extended_entities.media) { tweet.hasImage = true; } + // console.log(tweet.text); + // console.log(tweet.text.length); + // console.log(tweet.truncated); return tweet; } @@ -411,3 +575,32 @@ function getUserIdFromCredential(credential: firebase.auth.OAuthCredential) { const match = credential.accessToken?.match('[0-9]+'); return match && match.length ? match[0] : null; } + +// Format a millisecond-based timestamp into the yyyymmddhhmm date format +// suitable for the Enteprise Twitter API, as defined in: +// https://developer.twitter.com/en/docs/tweets/search/api-reference/enterprise-search +function formatTimestamp(ms: number): string { + const date = new Date(ms); + const MM = date.getUTCMonth() + 1; // getMonth() is zero-based + const dd = date.getUTCDate(); + const hh = date.getUTCHours(); + const mm = date.getUTCMinutes(); + + return ( + `${date.getFullYear()}` + + `${(MM > 9 ? '' : '0') + MM}` + + `${(dd > 9 ? '' : '0') + dd}` + + `${(hh > 9 ? '' : '0') + hh}` + + `${(mm > 9 ? '' : '0') + mm}` + ); +} + +// Format a millisecond-based timestamp into the YYYY-MM-DDTHH:mm:ssZ date format +// suitable for the v2 Twitter API, as defined in: +// https://developer.twitter.com/en/docs/twitter-api/tweets/search/api-reference/get-tweets-search-all +function formatTimestampForV2(ms: number): string { + const date = new Date(ms); + // Remove milliseconds from the ISO string (e.g. + // from 2022-10-12T23:09:43.430Z to 2022-10-12T23:09:43Z. + return date.toISOString().substring(0, 19) + 'Z'; +} diff --git a/src/server/serving.ts b/src/server/serving.ts index 6dd93a8..dda51fb 100644 --- a/src/server/serving.ts +++ b/src/server/serving.ts @@ -75,6 +75,7 @@ export interface TwitterApiCredentials { accountName: string; appKey: string; appToken: string; + bearerToken?: string; password: string; username: string; } From 2c136d2cf6ebc8525fe9deb24c083bc5362bf237 Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:59:27 -0500 Subject: [PATCH 2/9] Change AppEngine runtime to nodejs12 and update .gcloudignore --- .gcloudignore | 5 ++++- app.yaml | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gcloudignore b/.gcloudignore index a3f0c76..3d36e62 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -14,4 +14,7 @@ .gitignore # Node.js dependencies: -node_modules/ \ No newline at end of file +node_modules/ + +# Miscellaneous +.angular/cache \ No newline at end of file diff --git a/app.yaml b/app.yaml index c0a65ed..8f6a197 100644 --- a/app.yaml +++ b/app.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -runtime: nodejs10 +runtime: nodejs12 service: default instance_class: F4_1G @@ -25,7 +25,7 @@ automatic_scaling: # Required for min_idle_instances! inbound_services: -- warmup + - warmup env_variables: NODE_ENV: production From d832b003f23bd76ed1964646dc1ae8e95a4ae936 Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:59:52 -0500 Subject: [PATCH 3/9] Change @types/node to compatible version --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2a0fbff..cc00250 100644 --- a/package-lock.json +++ b/package-lock.json @@ -88,7 +88,7 @@ "@types/jasmine": "^3.10.3", "@types/jasminewd2": "~2.0.10", "@types/jspdf": "^1.3.3", - "@types/node": "^17.0.21", + "@types/node": "17.0.10", "@types/supertest": "^2.0.11", "@typescript-eslint/eslint-plugin": "5.14.0", "@typescript-eslint/parser": "5.14.0", @@ -6000,9 +6000,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "17.0.21", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", - "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" + "version": "17.0.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.10.tgz", + "integrity": "sha512-S/3xB4KzyFxYGCppyDt68yzBU9ysL88lSdIah4D6cptdcltc4NCPCAMc0+PCpg/lLIyC7IPvj2Z52OJWeIUkog==" }, "node_modules/@types/normalize-package-data": { "version": "2.4.1", @@ -29138,9 +29138,9 @@ "dev": true }, "@types/node": { - "version": "17.0.21", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", - "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" + "version": "17.0.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.10.tgz", + "integrity": "sha512-S/3xB4KzyFxYGCppyDt68yzBU9ysL88lSdIah4D6cptdcltc4NCPCAMc0+PCpg/lLIyC7IPvj2Z52OJWeIUkog==" }, "@types/normalize-package-data": { "version": "2.4.1", diff --git a/package.json b/package.json index 1aed3a1..a3c6529 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ "@types/jasmine": "^3.10.3", "@types/jasminewd2": "~2.0.10", "@types/jspdf": "^1.3.3", - "@types/node": "^17.0.21", + "@types/node": "17.0.10", "@types/supertest": "^2.0.11", "@typescript-eslint/eslint-plugin": "5.14.0", "@typescript-eslint/parser": "5.14.0", From 93e31608727485578c2f790b6bb04e4661cf75fa Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Mon, 28 Nov 2022 17:02:01 -0500 Subject: [PATCH 4/9] Clean up code --- src/app/social-media-item.service.ts | 6 +- src/common-types.ts | 32 ++-- src/server/middleware/twitter.middleware.ts | 201 +++++++++++--------- 3 files changed, 131 insertions(+), 108 deletions(-) diff --git a/src/app/social-media-item.service.ts b/src/app/social-media-item.service.ts index d9c1dba..5a5cb38 100644 --- a/src/app/social-media-item.service.ts +++ b/src/app/social-media-item.service.ts @@ -127,9 +127,9 @@ export class SocialMediaItemService { return from( this.twitterApiService.getTweets({ startDateTimeMs, - // Subtract 1 minute from the end time because the Twitter API - // sometimes returns an error if we request data for the most recent - // minute of time. + // Subtract 1 minute from the end time because the Twitter API sometimes + // returns an error if we request data for the most recent minute of + // time. endDateTimeMs: endDateTimeMs - 60000, }) ).pipe(map((response: GetTweetsResponse) => response.tweets)); diff --git a/src/common-types.ts b/src/common-types.ts index aaa2b86..658c3cb 100644 --- a/src/common-types.ts +++ b/src/common-types.ts @@ -128,6 +128,8 @@ export interface TwitterApiResponse { results: TweetObject[]; } +// Tweet object format returned by the Enterprise Twitter API. +// // From twitter documentation: When ingesting Tweet data the main object is the // Tweet Object, which is a parent object to several child objects. For // example, all Tweets include a User object that describes who authored the @@ -167,19 +169,31 @@ export interface TweetObject { source?: string; } +// Tweet object format returned by the v2 Twitter API. +// +// See +// https://developer.twitter.com/en/docs/twitter-api/data-dictionary/introduction +// for more details. export interface V2TweetObject { - attachments: V2Attachments; + attachments?: V2Attachments; author_id: string; created_at: string; - entities: V2Entities; + entities?: V2Entities; id: string; lang: string; public_metrics: V2PublicMetrics; - referenced_tweets: V2ReferencedTweet[]; - source: string; + referenced_tweets?: V2ReferencedTweet[]; + source?: string; text: string; } +interface V2Entities { + hashtags?: V2Hashtags[]; + mentions?: V2Mentions[]; + referenced_tweets?: V2ReferencedTweet[]; + urls?: V2Url[]; +} + interface V2PublicMetrics { like_count: number; quote_count: number; @@ -192,13 +206,6 @@ interface V2ReferencedTweet { type: string; } -interface V2Entities { - hashtags?: V2Hashtags[]; - mentions?: V2Mentions[]; - referenced_tweets?: V2ReferencedTweet[]; - urls?: V2Url[]; -} - interface V2Mentions { start: number; end: number; @@ -206,7 +213,7 @@ interface V2Mentions { id: string; } -interface V2Hashtags { +export interface V2Hashtags { start: number; end: number; tag: string; @@ -241,7 +248,6 @@ interface V2Attachments { media_keys: string[]; } -// TODO: Maybe remove all unused fields? export interface TwitterUser { id_str: string; screen_name: string; diff --git a/src/server/middleware/twitter.middleware.ts b/src/server/middleware/twitter.middleware.ts index b57115b..207fbf3 100644 --- a/src/server/middleware/twitter.middleware.ts +++ b/src/server/middleware/twitter.middleware.ts @@ -30,6 +30,7 @@ import { MuteTwitterUsersRequest, MuteTwitterUsersResponse, Tweet, + TweetEntities, TweetHashtag, TweetMedia, TweetObject, @@ -60,29 +61,19 @@ export async function getTweets( ) { let twitterDataPromise: Promise; - // if (fs.existsSync('src/server/twitter_sample_results.json')) { - // twitterDataPromise = loadLocalTwitterData(); - // } else if (enterpriseSearchCredentialsAreValid(apiCredentials)) { - // twitterDataPromise = loadTwitterData(apiCredentials, req.body); - // } else if (v2SearchCredentialsAreValid(apiCredentials)) { - // twitterDataPromise = loadTwitterDataV2(apiCredentials, req.body); - // } else { - // res.send(new Error('Invalid Search API credentials')); - // return; - // } - - // const enterpriseResults = ( - // await loadTwitterData(apiCredentials, req.body) - // ).results.map(parseTweet); - // const v2Results = ( - // await loadTwitterDataV2(apiCredentials, req.body) - // ).results.map(parseTweet); - - // console.log(JSON.stringify(enterpriseResults)); - // console.log(JSON.stringify(v2Results)); + if (fs.existsSync('src/server/twitter_sample_results.json')) { + twitterDataPromise = loadLocalTwitterData(); + } else if (v2SearchCredentialsAreValid(apiCredentials)) { + twitterDataPromise = loadTwitterDataV2(apiCredentials, req.body); + } else if (enterpriseSearchCredentialsAreValid(apiCredentials)) { + twitterDataPromise = loadTwitterData(apiCredentials, req.body); + } else { + res.send(new Error('No valid Twitter API credentials')); + return; + } try { - const twitterData = await loadTwitterData(apiCredentials, req.body); + const twitterData = await twitterDataPromise; const tweets = twitterData.results.map(parseTweet); res.send({ tweets, nextPageToken: twitterData.next } as GetTweetsResponse); } catch (e) { @@ -331,83 +322,48 @@ function loadTwitterDataV2( if (!user) { throw new Error('No user credentials in GetTweetsRequest'); } + + const params = { + // Include next_token if it's part of the request. + ...(request.nextPageToken && { next_token: request.nextPageToken }), + ...{ + query: `(@${user} OR url:twitter.com/${user}) -from:${user} -is:retweet`, + max_results: BATCH_SIZE, + 'user.fields': 'id,name,username,profile_image_url,verified', + expansions: 'author_id,attachments.media_keys,referenced_tweets.id', + start_time: formatTimestampForV2(request.startDateTimeMs), + end_time: formatTimestampForV2(request.endDateTimeMs), + 'media.fields': 'url,type', + 'tweet.fields': + 'attachments,created_at,id,entities,lang,public_metrics,source,text', + }, + }; + return axios .get(requestUrl, { headers: { authorization: `Bearer ${credentials.bearerToken}`, }, - params: { - query: `(@${user} OR url:twitter.com/${user}) -from:${user} -is:retweet`, - max_results: BATCH_SIZE, - 'user.fields': 'id,name,username,profile_image_url,verified', - expansions: 'author_id,attachments.media_keys,referenced_tweets.id', - start_time: formatTimestampForV2(request.startDateTimeMs), - end_time: formatTimestampForV2(request.endDateTimeMs), - 'media.fields': 'url,type', - 'tweet.fields': - 'attachments,created_at,id,entities,lang,public_metrics,source,text', - }, + params, transformResponse: [ (data, _) => { - const response = JSON.parse(data); - const tweets: V2TweetObject[] = response.data ?? []; - const includes: V2Includes = response.includes ?? {}; - const toReturn = { + const parsed = JSON.parse(data); + const tweets: V2TweetObject[] = parsed.data ?? []; + const includes: V2Includes = parsed.includes ?? {}; + return { results: tweets.map((tweet) => packV2Tweet(tweet, includes)), - next: response.meta.next_token ?? '', + next: parsed.meta.next_token, }; - return toReturn; }, ], }) .then((response) => response.data); } -// Packs a Tweet object response from the V2 Search PI into the Enterprise -// Search API response object format. +// Packs a Tweet response object from the v2 Search API format into the +// Enterprise Search API format. function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { - const entities = { - hashtags: tweet.entities?.hashtags?.map( - (hashtag) => - { - indices: [hashtag.start, hashtag.end], - text: hashtag.tag, - } - ), - // TODO: Handle this - symbols: [], - urls: tweet.entities?.urls?.map( - (url) => - { - display_url: url.display_url, - expanded_url: url.extended_url, - indices: [url.start, url.end], - } - ), - user_mentions: tweet.entities.mentions?.map( - (mention) => - { - id_str: mention.id, - indices: [mention.start, mention.end], - screen_name: mention.username, - } - ), - media: tweet.attachments?.media_keys - ?.flatMap((media_key) => { - const media = includes.media?.find( - (media) => media.media_key === media_key - ); - if (!media) { - throw new Error('Unable to find media'); - } - return { - indices: [0, 0], // Indices not available in V2. - media_url: media.url ?? '', - type: media.type, - }; - }) - .filter((media) => media.type === 'photo'), - }; + const entities = packEntities(tweet, includes); const tweetObject: TweetObject = { created_at: tweet.created_at, @@ -417,10 +373,9 @@ function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { // entities field is. entities: entities, extended_entities: entities, - // No display_text_range, truncated, or extended_tweet fields because the - // V2 API does not truncate the text. + // `display_text_range` omitted because v2 does not truncate the text. favorite_count: tweet.public_metrics.like_count, - // favorited is not available in V2 API. + // `favorited` omitted because it is not available in v2.. in_reply_to_status_id: tweet.entities?.referenced_tweets?.find( (tweet) => tweet.type === 'replied_to' )?.id, @@ -433,8 +388,8 @@ function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { }; if (tweetObject.truncated) { - // The Enteprise API populates the extended_tweet field if the tweet is - // truncated. + // Enteprise populates the extended_tweet field if the tweet is truncated, + // so we add it manually here for consistency. tweetObject.extended_tweet = { full_text: tweetObject.text, display_text_range: [0, 0], // Indices not available in v2. @@ -445,7 +400,72 @@ function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { return tweetObject; } +function packEntities( + tweet: V2TweetObject, + includes: V2Includes +): TweetEntities { + const entities: TweetEntities = {}; + + if (tweet.entities && tweet.entities.hashtags) { + entities.hashtags = tweet.entities.hashtags.map( + (hashtag) => + { + indices: [hashtag.start, hashtag.end], + text: hashtag.tag, + } + ); + } + + if (tweet.entities && tweet.entities.urls) { + entities.urls = tweet.entities.urls.map( + (url) => + { + display_url: url.display_url, + expanded_url: url.extended_url, + indices: [url.start, url.end], + } + ); + } + + if (tweet.entities && tweet.entities.mentions) { + entities.user_mentions = tweet.entities.mentions.map( + (mention) => + { + id_str: mention.id, + indices: [mention.start, mention.end], + screen_name: mention.username, + } + ); + } + + if (tweet.attachments && tweet.attachments.media_keys) { + entities.media = tweet.attachments.media_keys + .flatMap((media_key) => { + // v2 includes the media fields (like media_url) in a separate + // `includes` object, so we search there for the media item in question. + const media = includes.media?.find( + (media) => media.media_key === media_key + ); + // This shouldn't happen, but we throw an error if it does. + if (!media) { + throw new Error('Unable to find media'); + } + return { + indices: [0, 0], // Indices not available in v2. + media_url: media.url, + type: media.type, + }; + }) + // Filter for photos because that's all we display. + .filter((media) => media.type === 'photo'); + } + + return entities; +} + function getUser(id: string, includes: V2Includes): TwitterUser { + // v2 includes the user fields (like profile_image_url) in a separate + // `includes` object, so we search there for the media item in question. const user = includes.users?.find((user) => user.id === id); if (!user) { throw new Error('Unable to find user'); @@ -564,9 +584,6 @@ function parseTweet(tweetObject: TweetObject): Tweet { if (tweetObject.extended_entities && tweetObject.extended_entities.media) { tweet.hasImage = true; } - // console.log(tweet.text); - // console.log(tweet.text.length); - // console.log(tweet.truncated); return tweet; } @@ -595,8 +612,8 @@ function formatTimestamp(ms: number): string { ); } -// Format a millisecond-based timestamp into the YYYY-MM-DDTHH:mm:ssZ date format -// suitable for the v2 Twitter API, as defined in: +// Format a millisecond-based timestamp into the YYYY-MM-DDTHH:mm:ssZ date +// format suitable for the v2 Twitter API, as defined in: // https://developer.twitter.com/en/docs/twitter-api/tweets/search/api-reference/get-tweets-search-all function formatTimestampForV2(ms: number): string { const date = new Date(ms); From 020025db4970538b498b327c321ca4e93f371b37 Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Mon, 28 Nov 2022 17:02:11 -0500 Subject: [PATCH 5/9] Fix condition in TweetImageComponent template --- src/app/tweet-image/tweet-image.component.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/tweet-image/tweet-image.component.html b/src/app/tweet-image/tweet-image.component.html index c8bd696..e2dd1f6 100644 --- a/src/app/tweet-image/tweet-image.component.html +++ b/src/app/tweet-image/tweet-image.component.html @@ -16,7 +16,7 @@ + *ngIf="tweet.extended_entities?.media?.length"> @@ -24,9 +24,9 @@
+ *ngIf="tweet.extended_entities?.media?.length">
From 4cb1f8d0908c24054c3020f840b185ebd9d2043b Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Tue, 29 Nov 2022 13:42:28 -0500 Subject: [PATCH 6/9] Update server_config.template.json with bearerToken --- src/server/server_config.template.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server/server_config.template.json b/src/server/server_config.template.json index ac780d1..3cc0dd5 100644 --- a/src/server/server_config.template.json +++ b/src/server/server_config.template.json @@ -6,6 +6,9 @@ "twitterApiCredentials": { "accountName": "", "username": "", - "password": "" + "password": "", + "appKey": "", + "appToken": "", + "bearerToken": "" } } From 755af33f4f8b607f4f9172925ac963ae18382e8f Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Tue, 29 Nov 2022 13:42:35 -0500 Subject: [PATCH 7/9] Update documentation --- docs/1_setup.md | 64 ++++++++++++++++++++++++++++--------------- docs/2_development.md | 32 ++++++++++++++++------ 2 files changed, 65 insertions(+), 31 deletions(-) diff --git a/docs/1_setup.md b/docs/1_setup.md index b2b6122..066743d 100644 --- a/docs/1_setup.md +++ b/docs/1_setup.md @@ -18,40 +18,60 @@ of developers. ## 1. Get access to Twitter APIs -**NOTE: The full suite of Twitter APIs the app uses require additional access -beyond the default Twitter API [Essential access -level](https://developer.twitter.com/en/docs/twitter-api/getting-started/about-twitter-api). -The [Enterprise -search](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview) -additionally requires an [enterprise -account](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview).** - -**NOTE: We plan to migrate the Enterprise Full-Archive Search API to the v2 Search Tweets -in the future. We will update this documentation accordingly.** - The app makes use of several Twitter APIs, including: -- [The Enterprise Full-Archive Search - API](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview) to fetch - tweets directed at the logged in user -- The v2 [blocks](https://developer.twitter.com/en/docs/twitter-api/users/blocks/introduction) +- The [Enterprise Full-Archive Search + API](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview) + **or** the [v2 Full-Archive Search + endpoint](https://developer.twitter.com/en/docs/twitter-api/tweets/search/quick-start/full-archive-search) + to fetch tweets directed at the logged in user +- The v2 + [blocks](https://developer.twitter.com/en/docs/twitter-api/users/blocks/introduction) endpoint to block users on behalf of the authenticated user -- The v2 [mutes](https://developer.twitter.com/en/docs/twitter-api/users/mutes/introduction) +- The v2 + [mutes](https://developer.twitter.com/en/docs/twitter-api/users/mutes/introduction) endpoint to mute users on behalf of the authenticated user - The v2 [hide replies](https://developer.twitter.com/en/docs/twitter-api/tweets/hide-replies/introduction) endpoint to hide replies on behalf of the authenticated user To support all this functionality, you'll need to [get access to the Twitter -API](https://developer.twitter.com/en/docs/twitter-api/getting-started/getting-access-to-the-twitter-api) -and the Enterprise Full-Archive Search API. +API](https://developer.twitter.com/en/docs/twitter-api/getting-started/getting-access-to-the-twitter-api). -Once granted, take note of the: +**NOTE: The full suite of Twitter APIs the app uses require additional access +beyond the default Twitter API [Essential access +level](https://developer.twitter.com/en/docs/twitter-api/getting-started/about-twitter-api). +The [Enterprise +search](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview) +API requires an [enterprise +account](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview), +while v2 Full-Archive Search requires [academic +access](https://developer.twitter.com/en/products/twitter-api/academic-research).** -- Account name, app key, and app secret for your Twitter API developer account -- Username and password for your Enterpise Full-Archive Search API account +Once granted access, take note of the: -You'll need both sets of credentials later on. +- Account name, app key, and app secret for your Twitter API developer account +- If using the Enterprise Full-Archive Search API, the Username and password for + your enterprise account +- If using the v2 Full-Archive Search endpoint, the Twitter bearer token for + your app + +You'll need these credentials later on. + +### Enterprise Full-Archive Search vs. v2 Full-Archive Search + +The tool is implemented in a way that either API can be used. While both APIs +offer similar functionality, there are key differences in rate limits. We refer +users to [Twitter's +comparison](https://developer.twitter.com/en/docs/twitter-api/tweets/search/migrate) +for more details. You may also see minute differences in: + +- Which tweets are fetched. This is due to differences in granularity for + timestamp format each API supports (YYYYMMDD for Enterprise and + YYYY-MM-DDTHH:mm:ssZ for v2). +- The order the tweets are displayed when sorted by "Priority". This is due to + small differences in how the APIs return the tweet text, which causes some + variation in the Perspective API scores for the text. ## 2. Create a Google Cloud Platform (GCP) project diff --git a/docs/2_development.md b/docs/2_development.md index 918dc71..22041ac 100644 --- a/docs/2_development.md +++ b/docs/2_development.md @@ -148,13 +148,12 @@ The required fields are: The optional fields are: -- `twitterApiCredentials`: Your credentials for the Twitter APIs. The server - expect this field to be an object with `accountName`, `username`, and - `password` fields for the Enterprise Search API and `appKey` and `appToken` - for the Standard API. +- `twitterApiCredentials`: Your credentials for the Twitter APIs. -All together, your config should look something like the config below, with the -relevant credentials and key values replaced. +All together, your config should look something like one of the two configs +below, with the relevant credentials and key values replaced. + +### If using the Enteprise Full-Archive Search API: ```json { @@ -166,12 +165,27 @@ relevant credentials and key values replaced. "accountName": "{TWITTER_API_ACCOUNT_NAME}", "username": "{TWITTER_API_USERNAME}", "password": "{TWITTER_API_PASSWORD}", - "appKey": "{APP_KEY}", - "appToken": "{APP_TOKEN}" + "appKey": "{TWITTER_APP_KEY}", + "appToken": "{TWITTER_APP_TOKEN}" } } ``` +### If using the v2 Full-Archive Search endpoint: + +````json +{ + "port": "3000", + "staticPath": "dist/harassment-manager", + "googleCloudApiKey": "{YOUR_GOOGLE_CLOUD_API_KEY}", + "cloudProjectId": "{YOUR_GOOGLE_CLOUD_PROJECTID}", + "twitterApiCredentials": { + "appKey": "{TWITTER_APP_KEY}", + "appToken": "{TWITTER_APP_TOKEN}", + "bearerToken": "{TWITTER_APP_BEARER_TOKEN}" + } +} + ## 7. (Optional) Enable Google Analytics If you created a Google Analytics property in [setup](1_setup.md) and want to @@ -192,7 +206,7 @@ To build and run the app and a local development server, run ```shell npm run build:all:dev && npm run start:dev-server -``` +```` To build and run the app and a local production server, run From d67cb7cb895df793aa468c9fec7bb2f86ee082a9 Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Tue, 29 Nov 2022 15:43:55 -0500 Subject: [PATCH 8/9] Make a few tweaks to TwitterApiCredentials and documentation --- docs/2_development.md | 3 --- src/server/middleware/twitter.middleware.ts | 4 ++-- src/server/serving.ts | 11 ++++++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/2_development.md b/docs/2_development.md index 22041ac..2b0e88c 100644 --- a/docs/2_development.md +++ b/docs/2_development.md @@ -145,9 +145,6 @@ The required fields are: be the server-side key created in [setup](1_setup.md) in GCP [Credentials](https://console.cloud.google.com/apis/credentials) - `cloudProjectId`: Your Google Cloud project ID - -The optional fields are: - - `twitterApiCredentials`: Your credentials for the Twitter APIs. All together, your config should look something like one of the two configs diff --git a/src/server/middleware/twitter.middleware.ts b/src/server/middleware/twitter.middleware.ts index 207fbf3..55bf015 100644 --- a/src/server/middleware/twitter.middleware.ts +++ b/src/server/middleware/twitter.middleware.ts @@ -298,8 +298,8 @@ function loadTwitterData( } const auth: AxiosBasicCredentials = { - username: credentials!.username, - password: credentials!.password, + username: credentials.username!, + password: credentials.password!, }; return axios diff --git a/src/server/serving.ts b/src/server/serving.ts index dda51fb..0961957 100644 --- a/src/server/serving.ts +++ b/src/server/serving.ts @@ -72,12 +72,17 @@ export interface Config { } export interface TwitterApiCredentials { - accountName: string; + // The below three fields are necessary if using Enterprise Full-Archive + // Search. + accountName?: string; + password?: string; + username?: string; + // The below two fields are necessary for the Blocks, Mutes, and Hide Replies + // APIs. appKey: string; appToken: string; + // Necessary if using v2 Full-Archive Search. bearerToken?: string; - password: string; - username: string; } export interface WebAppCredentials { From c2f483e037d750fad63f435f5284dab5bc712918 Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Wed, 30 Nov 2022 16:42:51 -0500 Subject: [PATCH 9/9] Address PR review comments --- docs/1_setup.md | 5 ++- docs/2_development.md | 15 +++++-- src/app/test_constants.ts | 4 -- src/common-types.ts | 2 - src/server/middleware/twitter.middleware.ts | 48 +++++++++++---------- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/docs/1_setup.md b/docs/1_setup.md index 066743d..c7ce3ee 100644 --- a/docs/1_setup.md +++ b/docs/1_setup.md @@ -70,8 +70,9 @@ for more details. You may also see minute differences in: timestamp format each API supports (YYYYMMDD for Enterprise and YYYY-MM-DDTHH:mm:ssZ for v2). - The order the tweets are displayed when sorted by "Priority". This is due to - small differences in how the APIs return the tweet text, which causes some - variation in the Perspective API scores for the text. + small differences in how we parse out the tweet text, which causes some + variation in the Perspective API scores for the text. See issue #19 for more + details. ## 2. Create a Google Cloud Platform (GCP) project diff --git a/docs/2_development.md b/docs/2_development.md index 2b0e88c..53827e4 100644 --- a/docs/2_development.md +++ b/docs/2_development.md @@ -145,7 +145,11 @@ The required fields are: be the server-side key created in [setup](1_setup.md) in GCP [Credentials](https://console.cloud.google.com/apis/credentials) - `cloudProjectId`: Your Google Cloud project ID -- `twitterApiCredentials`: Your credentials for the Twitter APIs. +- `twitterApiCredentials`: Your credentials for the Twitter APIs. For Enterprise + Full-Archive search, Twitter will provide you with the credentials. All other + API credentials should be available on the Twitter [Developer + Portal](https://developer.twitter.com/portal) under "Keys and Tokens" for your + app and project. All together, your config should look something like one of the two configs below, with the relevant credentials and key values replaced. @@ -170,7 +174,7 @@ below, with the relevant credentials and key values replaced. ### If using the v2 Full-Archive Search endpoint: -````json +```json { "port": "3000", "staticPath": "dist/harassment-manager", @@ -182,6 +186,7 @@ below, with the relevant credentials and key values replaced. "bearerToken": "{TWITTER_APP_BEARER_TOKEN}" } } +``` ## 7. (Optional) Enable Google Analytics @@ -203,7 +208,7 @@ To build and run the app and a local development server, run ```shell npm run build:all:dev && npm run start:dev-server -```` +``` To build and run the app and a local production server, run @@ -237,3 +242,7 @@ We maintain a [CircleCI](https://circleci.com/) configuration in is pushed to this GitHub repository. You can choose to use the same configuration for your own CircleCI setup if you'd like or remove the configuration in favor of another CI solution or none at all. + +``` + +``` diff --git a/src/app/test_constants.ts b/src/app/test_constants.ts index ccda5dc..ded7ba9 100644 --- a/src/app/test_constants.ts +++ b/src/app/test_constants.ts @@ -37,7 +37,6 @@ export const TWITTER_ENTRIES: Array> = [ user_mentions: [], }, extended_tweet: { - display_text_range: [0, 279], entities: { hashtags: [ { @@ -110,7 +109,6 @@ export const TWITTER_ENTRIES: Array> = [ user_mentions: [], }, extended_tweet: { - display_text_range: [0, 213], entities: { hashtags: [ { @@ -189,7 +187,6 @@ export const TWITTER_ENTRIES: Array> = [ ], }, extended_tweet: { - display_text_range: [0, 203], entities: { hashtags: [], symbols: [], @@ -274,7 +271,6 @@ export const TWITTER_ENTRIES: Array> = [ user_mentions: [], }, extended_tweet: { - display_text_range: [0, 274], entities: { hashtags: [], symbols: [], diff --git a/src/common-types.ts b/src/common-types.ts index 658c3cb..33b63c5 100644 --- a/src/common-types.ts +++ b/src/common-types.ts @@ -153,7 +153,6 @@ export interface TweetObject { // directed to the extended_entities section. entities?: TweetEntities; - display_text_range?: number[]; truncated?: boolean; extended_tweet?: ExtendedTweet; @@ -323,7 +322,6 @@ export interface TweetHashtag { // For tweets above 140 characters. interface ExtendedTweet { full_text: string; - display_text_range: number[]; entities: TweetEntities; } diff --git a/src/server/middleware/twitter.middleware.ts b/src/server/middleware/twitter.middleware.ts index 55bf015..42f784e 100644 --- a/src/server/middleware/twitter.middleware.ts +++ b/src/server/middleware/twitter.middleware.ts @@ -64,8 +64,10 @@ export async function getTweets( if (fs.existsSync('src/server/twitter_sample_results.json')) { twitterDataPromise = loadLocalTwitterData(); } else if (v2SearchCredentialsAreValid(apiCredentials)) { + console.log('Fetching tweets using the Enterprise Full-Archive Search API'); twitterDataPromise = loadTwitterDataV2(apiCredentials, req.body); } else if (enterpriseSearchCredentialsAreValid(apiCredentials)) { + console.log('Fetching tweets using the v2 Full-Archive Search API'); twitterDataPromise = loadTwitterData(apiCredentials, req.body); } else { res.send(new Error('No valid Twitter API credentials')); @@ -351,7 +353,9 @@ function loadTwitterDataV2( const tweets: V2TweetObject[] = parsed.data ?? []; const includes: V2Includes = parsed.includes ?? {}; return { - results: tweets.map((tweet) => packV2Tweet(tweet, includes)), + results: tweets.map((tweet) => + packV2TweetAsEnterprise(tweet, includes) + ), next: parsed.meta.next_token, }; }, @@ -362,7 +366,10 @@ function loadTwitterDataV2( // Packs a Tweet response object from the v2 Search API format into the // Enterprise Search API format. -function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { +function packV2TweetAsEnterprise( + tweet: V2TweetObject, + includes: V2Includes +): TweetObject { const entities = packEntities(tweet, includes); const tweetObject: TweetObject = { @@ -373,7 +380,6 @@ function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { // entities field is. entities: entities, extended_entities: entities, - // `display_text_range` omitted because v2 does not truncate the text. favorite_count: tweet.public_metrics.like_count, // `favorited` omitted because it is not available in v2.. in_reply_to_status_id: tweet.entities?.referenced_tweets?.find( @@ -392,7 +398,6 @@ function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { // so we add it manually here for consistency. tweetObject.extended_tweet = { full_text: tweetObject.text, - display_text_range: [0, 0], // Indices not available in v2. entities, }; } @@ -406,7 +411,7 @@ function packEntities( ): TweetEntities { const entities: TweetEntities = {}; - if (tweet.entities && tweet.entities.hashtags) { + if (tweet.entities?.hashtags) { entities.hashtags = tweet.entities.hashtags.map( (hashtag) => { @@ -416,7 +421,7 @@ function packEntities( ); } - if (tweet.entities && tweet.entities.urls) { + if (tweet.entities?.urls) { entities.urls = tweet.entities.urls.map( (url) => { @@ -427,7 +432,7 @@ function packEntities( ); } - if (tweet.entities && tweet.entities.mentions) { + if (tweet.entities?.mentions) { entities.user_mentions = tweet.entities.mentions.map( (mention) => { @@ -438,7 +443,7 @@ function packEntities( ); } - if (tweet.attachments && tweet.attachments.media_keys) { + if (tweet.attachments?.media_keys) { entities.media = tweet.attachments.media_keys .flatMap((media_key) => { // v2 includes the media fields (like media_url) in a separate @@ -471,11 +476,11 @@ function getUser(id: string, includes: V2Includes): TwitterUser { throw new Error('Unable to find user'); } return { - id_str: user?.id, - profile_image_url: user?.profile_image_url, - name: user?.name, - screen_name: user?.username, - verified: user?.verified, + id_str: user.id, + profile_image_url: user.profile_image_url, + name: user.name, + screen_name: user.username, + verified: user.verified, }; } @@ -549,7 +554,6 @@ function parseTweet(tweetObject: TweetObject): Tweet { const tweet: Tweet = { created_at: tweetObject.created_at, date: new Date(), - display_text_range: tweetObject.display_text_range, entities: tweetObject.entities, extended_entities: tweetObject.extended_entities, extended_tweet: tweetObject.extended_tweet, @@ -598,17 +602,17 @@ function getUserIdFromCredential(credential: firebase.auth.OAuthCredential) { // https://developer.twitter.com/en/docs/tweets/search/api-reference/enterprise-search function formatTimestamp(ms: number): string { const date = new Date(ms); - const MM = date.getUTCMonth() + 1; // getMonth() is zero-based - const dd = date.getUTCDate(); - const hh = date.getUTCHours(); - const mm = date.getUTCMinutes(); + const month = date.getUTCMonth() + 1; // getMonth() is zero-based + const day = date.getUTCDate(); + const hours = date.getUTCHours(); + const minutes = date.getUTCMinutes(); return ( `${date.getFullYear()}` + - `${(MM > 9 ? '' : '0') + MM}` + - `${(dd > 9 ? '' : '0') + dd}` + - `${(hh > 9 ? '' : '0') + hh}` + - `${(mm > 9 ? '' : '0') + mm}` + `${(month > 9 ? '' : '0') + month}` + + `${(day > 9 ? '' : '0') + day}` + + `${(hours > 9 ? '' : '0') + hours}` + + `${(minutes > 9 ? '' : '0') + minutes}` ); }