From c2f483e037d750fad63f435f5284dab5bc712918 Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Wed, 30 Nov 2022 16:42:51 -0500 Subject: [PATCH] Address PR review comments --- docs/1_setup.md | 5 ++- docs/2_development.md | 15 +++++-- src/app/test_constants.ts | 4 -- src/common-types.ts | 2 - src/server/middleware/twitter.middleware.ts | 48 +++++++++++---------- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/docs/1_setup.md b/docs/1_setup.md index 066743d..c7ce3ee 100644 --- a/docs/1_setup.md +++ b/docs/1_setup.md @@ -70,8 +70,9 @@ for more details. You may also see minute differences in: timestamp format each API supports (YYYYMMDD for Enterprise and YYYY-MM-DDTHH:mm:ssZ for v2). - The order the tweets are displayed when sorted by "Priority". This is due to - small differences in how the APIs return the tweet text, which causes some - variation in the Perspective API scores for the text. + small differences in how we parse out the tweet text, which causes some + variation in the Perspective API scores for the text. See issue #19 for more + details. ## 2. Create a Google Cloud Platform (GCP) project diff --git a/docs/2_development.md b/docs/2_development.md index 2b0e88c..53827e4 100644 --- a/docs/2_development.md +++ b/docs/2_development.md @@ -145,7 +145,11 @@ The required fields are: be the server-side key created in [setup](1_setup.md) in GCP [Credentials](https://console.cloud.google.com/apis/credentials) - `cloudProjectId`: Your Google Cloud project ID -- `twitterApiCredentials`: Your credentials for the Twitter APIs. +- `twitterApiCredentials`: Your credentials for the Twitter APIs. For Enterprise + Full-Archive search, Twitter will provide you with the credentials. All other + API credentials should be available on the Twitter [Developer + Portal](https://developer.twitter.com/portal) under "Keys and Tokens" for your + app and project. All together, your config should look something like one of the two configs below, with the relevant credentials and key values replaced. @@ -170,7 +174,7 @@ below, with the relevant credentials and key values replaced. ### If using the v2 Full-Archive Search endpoint: -````json +```json { "port": "3000", "staticPath": "dist/harassment-manager", @@ -182,6 +186,7 @@ below, with the relevant credentials and key values replaced. "bearerToken": "{TWITTER_APP_BEARER_TOKEN}" } } +``` ## 7. (Optional) Enable Google Analytics @@ -203,7 +208,7 @@ To build and run the app and a local development server, run ```shell npm run build:all:dev && npm run start:dev-server -```` +``` To build and run the app and a local production server, run @@ -237,3 +242,7 @@ We maintain a [CircleCI](https://circleci.com/) configuration in is pushed to this GitHub repository. You can choose to use the same configuration for your own CircleCI setup if you'd like or remove the configuration in favor of another CI solution or none at all. + +``` + +``` diff --git a/src/app/test_constants.ts b/src/app/test_constants.ts index ccda5dc..ded7ba9 100644 --- a/src/app/test_constants.ts +++ b/src/app/test_constants.ts @@ -37,7 +37,6 @@ export const TWITTER_ENTRIES: Array> = [ user_mentions: [], }, extended_tweet: { - display_text_range: [0, 279], entities: { hashtags: [ { @@ -110,7 +109,6 @@ export const TWITTER_ENTRIES: Array> = [ user_mentions: [], }, extended_tweet: { - display_text_range: [0, 213], entities: { hashtags: [ { @@ -189,7 +187,6 @@ export const TWITTER_ENTRIES: Array> = [ ], }, extended_tweet: { - display_text_range: [0, 203], entities: { hashtags: [], symbols: [], @@ -274,7 +271,6 @@ export const TWITTER_ENTRIES: Array> = [ user_mentions: [], }, extended_tweet: { - display_text_range: [0, 274], entities: { hashtags: [], symbols: [], diff --git a/src/common-types.ts b/src/common-types.ts index 658c3cb..33b63c5 100644 --- a/src/common-types.ts +++ b/src/common-types.ts @@ -153,7 +153,6 @@ export interface TweetObject { // directed to the extended_entities section. entities?: TweetEntities; - display_text_range?: number[]; truncated?: boolean; extended_tweet?: ExtendedTweet; @@ -323,7 +322,6 @@ export interface TweetHashtag { // For tweets above 140 characters. interface ExtendedTweet { full_text: string; - display_text_range: number[]; entities: TweetEntities; } diff --git a/src/server/middleware/twitter.middleware.ts b/src/server/middleware/twitter.middleware.ts index 55bf015..42f784e 100644 --- a/src/server/middleware/twitter.middleware.ts +++ b/src/server/middleware/twitter.middleware.ts @@ -64,8 +64,10 @@ export async function getTweets( if (fs.existsSync('src/server/twitter_sample_results.json')) { twitterDataPromise = loadLocalTwitterData(); } else if (v2SearchCredentialsAreValid(apiCredentials)) { + console.log('Fetching tweets using the Enterprise Full-Archive Search API'); twitterDataPromise = loadTwitterDataV2(apiCredentials, req.body); } else if (enterpriseSearchCredentialsAreValid(apiCredentials)) { + console.log('Fetching tweets using the v2 Full-Archive Search API'); twitterDataPromise = loadTwitterData(apiCredentials, req.body); } else { res.send(new Error('No valid Twitter API credentials')); @@ -351,7 +353,9 @@ function loadTwitterDataV2( const tweets: V2TweetObject[] = parsed.data ?? []; const includes: V2Includes = parsed.includes ?? {}; return { - results: tweets.map((tweet) => packV2Tweet(tweet, includes)), + results: tweets.map((tweet) => + packV2TweetAsEnterprise(tweet, includes) + ), next: parsed.meta.next_token, }; }, @@ -362,7 +366,10 @@ function loadTwitterDataV2( // Packs a Tweet response object from the v2 Search API format into the // Enterprise Search API format. -function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { +function packV2TweetAsEnterprise( + tweet: V2TweetObject, + includes: V2Includes +): TweetObject { const entities = packEntities(tweet, includes); const tweetObject: TweetObject = { @@ -373,7 +380,6 @@ function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { // entities field is. entities: entities, extended_entities: entities, - // `display_text_range` omitted because v2 does not truncate the text. favorite_count: tweet.public_metrics.like_count, // `favorited` omitted because it is not available in v2.. in_reply_to_status_id: tweet.entities?.referenced_tweets?.find( @@ -392,7 +398,6 @@ function packV2Tweet(tweet: V2TweetObject, includes: V2Includes): TweetObject { // so we add it manually here for consistency. tweetObject.extended_tweet = { full_text: tweetObject.text, - display_text_range: [0, 0], // Indices not available in v2. entities, }; } @@ -406,7 +411,7 @@ function packEntities( ): TweetEntities { const entities: TweetEntities = {}; - if (tweet.entities && tweet.entities.hashtags) { + if (tweet.entities?.hashtags) { entities.hashtags = tweet.entities.hashtags.map( (hashtag) => { @@ -416,7 +421,7 @@ function packEntities( ); } - if (tweet.entities && tweet.entities.urls) { + if (tweet.entities?.urls) { entities.urls = tweet.entities.urls.map( (url) => { @@ -427,7 +432,7 @@ function packEntities( ); } - if (tweet.entities && tweet.entities.mentions) { + if (tweet.entities?.mentions) { entities.user_mentions = tweet.entities.mentions.map( (mention) => { @@ -438,7 +443,7 @@ function packEntities( ); } - if (tweet.attachments && tweet.attachments.media_keys) { + if (tweet.attachments?.media_keys) { entities.media = tweet.attachments.media_keys .flatMap((media_key) => { // v2 includes the media fields (like media_url) in a separate @@ -471,11 +476,11 @@ function getUser(id: string, includes: V2Includes): TwitterUser { throw new Error('Unable to find user'); } return { - id_str: user?.id, - profile_image_url: user?.profile_image_url, - name: user?.name, - screen_name: user?.username, - verified: user?.verified, + id_str: user.id, + profile_image_url: user.profile_image_url, + name: user.name, + screen_name: user.username, + verified: user.verified, }; } @@ -549,7 +554,6 @@ function parseTweet(tweetObject: TweetObject): Tweet { const tweet: Tweet = { created_at: tweetObject.created_at, date: new Date(), - display_text_range: tweetObject.display_text_range, entities: tweetObject.entities, extended_entities: tweetObject.extended_entities, extended_tweet: tweetObject.extended_tweet, @@ -598,17 +602,17 @@ function getUserIdFromCredential(credential: firebase.auth.OAuthCredential) { // https://developer.twitter.com/en/docs/tweets/search/api-reference/enterprise-search function formatTimestamp(ms: number): string { const date = new Date(ms); - const MM = date.getUTCMonth() + 1; // getMonth() is zero-based - const dd = date.getUTCDate(); - const hh = date.getUTCHours(); - const mm = date.getUTCMinutes(); + const month = date.getUTCMonth() + 1; // getMonth() is zero-based + const day = date.getUTCDate(); + const hours = date.getUTCHours(); + const minutes = date.getUTCMinutes(); return ( `${date.getFullYear()}` + - `${(MM > 9 ? '' : '0') + MM}` + - `${(dd > 9 ? '' : '0') + dd}` + - `${(hh > 9 ? '' : '0') + hh}` + - `${(mm > 9 ? '' : '0') + mm}` + `${(month > 9 ? '' : '0') + month}` + + `${(day > 9 ? '' : '0') + day}` + + `${(hours > 9 ? '' : '0') + hours}` + + `${(minutes > 9 ? '' : '0') + minutes}` ); }