Adds substances collector

resources/armstrong.txt

@@ -0,0 +1,10 @@
+PgRBxsW26vY
+wzvZNQKowMU
+u1qf9TlFTl4
+t6OPh_a11QA
+ZxkULBtpF3s
+2uytJ2HQf8E
+u7kFqp_p09o
+HQjcPDg0UzY
+ttdQp8nuyo8
+N_0PSZ59Aws

resources/gay.txt

@@ -1,3 +1,8 @@
-6lYivLjHjKs
 Z8y0Sb3EUag
 tjfl4IGR7VA
+UgJA9yQY9ZA
+FaNnV8aooJQ
+oUarPvJSGYo
+CaffhJ7UjzU
+cSbkWvxfcto
+6lYivLjHjKs

resources/jones.txt

@@ -2,3 +2,9 @@ DkQpTdVK1cc
 H3JmeHKFEZU
 y1USv6QL-Dc
 rWyGWu186zY
+3ewja4q0z7s
+smh-qgB8s_M
+Twubt8xcqus
+orn94KTWwnw
+Jc3hRXq0eAw
+8pmDTbt157c

src/combiner/Combiner.java

@@ -30,13 +30,13 @@ public static void main(String[] args) throws JSONException, IOException {
 
         if(args != null) {
             // Collect Youtube comments and insert them to DB
-            /*YoutubeExporter.main(args);
-            Preprocessor.preprocessComments(args[4].split("=")[1], fc);*/
+            /* YoutubeExporter.main(args);
+            Preprocessor.preprocessComments(args[4].split("=")[1], fc); */
 
 
             // Collect tweets and insert them to DB
-            /*TwitterExporter.main(args);
-            Preprocessor.preprocessTweets(args[4].split("=")[1], fc);*/
+            TwitterExporter.main(args);
+            Preprocessor.preprocessTweets(args[4].split("=")[1], fc, args[5].split("=")[1]);
 
             /*fc.exportFrequencies(); //creates frequencies.txt - sorted alphabetically
             fc.exportFrequenciesByValue(); //creates frequenciesByValue.txt - sorted by frequencies (descending order)*/

src/combiner/Preprocessor.java

@@ -6,11 +6,13 @@
 import org.json.JSONException;
 import org.json.JSONObject;
 
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
+import java.util.*;
 
 /**
  * Class to process text (tweets and comments)
@@ -19,8 +21,29 @@
 
 public class Preprocessor {
 
-    private static ArrayList<String> stopwordsList = new ArrayList<>(Arrays.asList("a", "as", "able", "about", "above", "according", "accordingly", "across", "actually", "after", "afterwards", "again", "against", "aint", "all", "allow", "allows", "almost", "alone", "along", "already", "also", "although", "always", "am", "among", "amongst", "an", "and", "another", "any", "anybody", "anyhow", "anyone", "anything", "anyway", "anyways", "anywhere", "apart", "appear", "appreciate", "appropriate", "are", "arent", "around", "as", "aside", "ask", "asking", "associated", "at", "available", "away", "awfully", "be", "became", "because", "become", "becomes", "becoming", "been", "before", "beforehand", "behind", "being", "believe", "below", "beside", "besides", "best", "better", "between", "beyond", "both", "brief", "but", "by", "cmon", "cs", "came", "can", "cant", "cannot", "cant", "cause", "causes", "certain", "certainly", "changes", "clearly", "co", "com", "come", "comes", "concerning", "consequently", "consider", "considering", "contain", "containing", "contains", "corresponding", "could", "couldnt", "course", "currently", "definitely", "described", "despite", "did", "didnt", "different", "do", "does", "doesnt", "doing", "dont", "done", "down", "downwards", "during", "each", "edu", "eg", "eight", "either", "else", "elsewhere", "enough", "entirely", "especially", "et", "etc", "even", "ever", "every", "everybody", "everyone", "everything", "everywhere", "ex", "exactly", "example", "except", "far", "few", "ff", "fifth", "first", "five", "followed", "following", "follows", "for", "former", "formerly", "forth", "four", "from", "further", "furthermore", "get", "gets", "getting", "given", "gives", "go", "goes", "going", "gone", "got", "gotten", "greetings", "had", "hadnt", "happens", "hardly", "has", "hasnt", "have", "havent", "having", "he", "hes", "hello", "help", "hence", "her", "here", "heres", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "hi", "him", "himself", "his", "hither", "hopefully", "how", "howbeit", "however", "i", "id", "ill", "im", "ive", "ie", "if", "ignored", "immediate", "in", "inasmuch", "inc", "indeed", "indicate", "indicated", "indicates", "inner", "insofar", "instead", "into", "inward", "is", "isnt", "it", "itd", "itll", "its", "its", "itself", "just", "keep", "keeps", "kept", "know", "knows", "known", "last", "lately", "later", "latter", "latterly", "least", "less", "lest", "let", "lets", "like", "liked", "likely", "little", "look", "looking", "looks", "ltd", "mainly", "many", "may", "maybe", "me", "mean", "meanwhile", "merely", "might", "more", "moreover", "most", "mostly", "much", "must", "my", "myself", "name", "namely", "nd", "near", "nearly", "necessary", "need", "needs", "neither", "never", "nevertheless", "new", "next", "nine", "no", "nobody", "non", "none", "noone", "nor", "normally", "not", "nothing", "novel", "now", "nowhere", "obviously", "of", "off", "often", "oh", "ok", "okay", "old", "on", "once", "one", "ones", "only", "onto", "or", "other", "others", "otherwise", "ought", "our", "ours", "ourselves", "out", "outside", "over", "overall", "own", "particular", "particularly", "per", "perhaps", "placed", "please", "plus", "possible", "presumably", "probably", "provides", "que", "quite", "qv", "rather", "rd", "re", "really", "reasonably", "regarding", "regardless", "regards", "relatively", "respectively", "right", "said", "same", "saw", "say", "saying", "says", "second", "secondly", "see", "seeing", "seem", "seemed", "seeming", "seems", "seen", "self", "selves", "sensible", "sent", "serious", "seriously", "seven", "several", "shall", "she", "should", "shouldnt", "since", "six", "so", "some", "somebody", "somehow", "someone", "something", "sometime", "sometimes", "somewhat", "somewhere", "soon", "sorry", "specified", "specify", "specifying", "still", "sub", "such", "sup", "sure", "ts", "take", "taken", "tell", "tends", "th", "than", "thank", "thanks", "thanx", "that", "thats", "thats", "the", "their", "theirs", "them", "themselves", "then", "thence", "there", "theres", "thereafter", "thereby", "therefore", "therein", "theres", "thereupon", "these", "they", "theyd", "theyll", "theyre", "theyve", "think", "third", "this", "thorough", "thoroughly", "those", "though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "took", "toward", "towards", "tried", "tries", "truly", "try", "trying", "twice", "two", "un", "under", "unfortunately", "unless", "unlikely", "until", "unto", "up", "upon", "us", "use", "used", "useful", "uses", "using", "usually", "value", "various", "very", "via", "viz", "vs", "want", "wants", "was", "wasnt", "way", "we", "wed", "well", "were", "weve", "welcome", "well", "went", "were", "werent", "what", "whats", "whatever", "when", "whence", "whenever", "where", "wheres", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whos", "whoever", "whole", "whom", "whose", "why", "will", "willing", "wish", "with", "within", "without", "wont", "wonder", "would", "would", "wouldnt", "yes", "yet", "you", "youd", "youll", "youre", "youve", "your", "yours", "yourself", "yourselves", "zero"));
+    private static ArrayList<String> stopwordsList;
     private static MongoConnector mc;
+    private static ArrayList<String> substancesList;
+    private static HashSet<String> substancesSet;
+
+    static {
+        stopwordsList = new ArrayList<>(Arrays.asList("a", "as", "able", "about", "above", "according", "accordingly", "across", "actually", "after", "afterwards", "again", "against", "aint", "all", "allow", "allows", "almost", "alone", "along", "already", "also", "although", "always", "am", "among", "amongst", "an", "and", "another", "any", "anybody", "anyhow", "anyone", "anything", "anyway", "anyways", "anywhere", "apart", "appear", "appreciate", "appropriate", "are", "arent", "around", "as", "aside", "ask", "asking", "associated", "at", "available", "away", "awfully", "be", "became", "because", "become", "becomes", "becoming", "been", "before", "beforehand", "behind", "being", "believe", "below", "beside", "besides", "best", "better", "between", "beyond", "both", "brief", "but", "by", "cmon", "cs", "came", "can", "cant", "cannot", "cant", "cause", "causes", "certain", "certainly", "changes", "clearly", "co", "com", "come", "comes", "concerning", "consequently", "consider", "considering", "contain", "containing", "contains", "corresponding", "could", "couldnt", "course", "currently", "definitely", "described", "despite", "did", "didnt", "different", "do", "does", "doesnt", "doing", "dont", "done", "down", "downwards", "during", "each", "edu", "eg", "eight", "either", "else", "elsewhere", "enough", "entirely", "especially", "et", "etc", "even", "ever", "every", "everybody", "everyone", "everything", "everywhere", "ex", "exactly", "example", "except", "far", "few", "ff", "fifth", "first", "five", "followed", "following", "follows", "for", "former", "formerly", "forth", "four", "from", "further", "furthermore", "get", "gets", "getting", "given", "gives", "go", "goes", "going", "gone", "got", "gotten", "greetings", "had", "hadnt", "happens", "hardly", "has", "hasnt", "have", "havent", "having", "he", "hes", "hello", "help", "hence", "her", "here", "heres", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "hi", "him", "himself", "his", "hither", "hopefully", "how", "howbeit", "however", "i", "id", "ill", "im", "ive", "ie", "if", "ignored", "immediate", "in", "inasmuch", "inc", "indeed", "indicate", "indicated", "indicates", "inner", "insofar", "instead", "into", "inward", "is", "isnt", "it", "itd", "itll", "its", "its", "itself", "just", "keep", "keeps", "kept", "know", "knows", "known", "last", "lately", "later", "latter", "latterly", "least", "less", "lest", "let", "lets", "like", "liked", "likely", "little", "look", "looking", "looks", "ltd", "mainly", "many", "may", "maybe", "me", "mean", "meanwhile", "merely", "might", "more", "moreover", "most", "mostly", "much", "must", "my", "myself", "name", "namely", "nd", "near", "nearly", "necessary", "need", "needs", "neither", "never", "nevertheless", "new", "next", "nine", "no", "nobody", "non", "none", "noone", "nor", "normally", "not", "nothing", "novel", "now", "nowhere", "obviously", "of", "off", "often", "oh", "ok", "okay", "old", "on", "once", "one", "ones", "only", "onto", "or", "other", "others", "otherwise", "ought", "our", "ours", "ourselves", "out", "outside", "over", "overall", "own", "particular", "particularly", "per", "perhaps", "placed", "please", "plus", "possible", "presumably", "probably", "provides", "que", "quite", "qv", "rather", "rd", "re", "really", "reasonably", "regarding", "regardless", "regards", "relatively", "respectively", "right", "said", "same", "saw", "say", "saying", "says", "second", "secondly", "see", "seeing", "seem", "seemed", "seeming", "seems", "seen", "self", "selves", "sensible", "sent", "serious", "seriously", "seven", "several", "shall", "she", "should", "shouldnt", "since", "six", "so", "some", "somebody", "somehow", "someone", "something", "sometime", "sometimes", "somewhat", "somewhere", "soon", "sorry", "specified", "specify", "specifying", "still", "sub", "such", "sup", "sure", "ts", "take", "taken", "tell", "tends", "th", "than", "thank", "thanks", "thanx", "that", "thats", "thats", "the", "their", "theirs", "them", "themselves", "then", "thence", "there", "theres", "thereafter", "thereby", "therefore", "therein", "theres", "thereupon", "these", "they", "theyd", "theyll", "theyre", "theyve", "think", "third", "this", "thorough", "thoroughly", "those", "though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "took", "toward", "towards", "tried", "tries", "truly", "try", "trying", "twice", "two", "un", "under", "unfortunately", "unless", "unlikely", "until", "unto", "up", "upon", "us", "use", "used", "useful", "uses", "using", "usually", "value", "various", "very", "via", "viz", "vs", "want", "wants", "was", "wasnt", "way", "we", "wed", "well", "were", "weve", "welcome", "well", "went", "were", "werent", "what", "whats", "whatever", "when", "whence", "whenever", "where", "wheres", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whos", "whoever", "whole", "whom", "whose", "why", "will", "willing", "wish", "with", "within", "without", "wont", "wonder", "would", "would", "wouldnt", "yes", "yet", "you", "youd", "youll", "youre", "youve", "your", "yours", "yourself", "yourselves", "zero"));
+        substancesSet = new HashSet<>();
+        BufferedReader in;
+        try {
+            String filename = "./resources/substancesFinal.txt";
+            in = new BufferedReader(new FileReader(filename));
+            String line;
+            while((line = in.readLine()) != null)
+            {
+                substancesSet.add(line.toLowerCase());
+            }
+            in.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        substancesList = new ArrayList<>(substancesSet);
+    }
 
     /**
      * Processes all the comments' text
@@ -46,18 +69,49 @@ public static void preprocessComments(String collectionName, FrequencyCounter fc
      * @param fc A FrequencyCounter object for counting frequencies
      * @throws JSONException
      */
-    public static void preprocessTweets(String collectionName, FrequencyCounter fc) throws JSONException {
+    public static void preprocessTweets(String collectionName, FrequencyCounter fc, String substancesExist) throws JSONException {
         mc = new MongoConnector("localhost", 27017, collectionName);
         HashMap<ObjectId,JSONObject> tweets = mc.getTweets();
         String parsedTweet;
+        String tweet;
         for(ObjectId id : tweets.keySet()) {
-            parsedTweet = tweets.get(id).getString("text"); // Get tweet's text to process
-            parsedTweet = preprocessTweet(parsedTweet);
-            mc.insertParsedTweet(id,parsedTweet);
-            fc.insert(parsedTweet);//insert to treemaps
+            tweet= tweets.get(id).getString("text"); // Get tweet's text to process
+            parsedTweet = preprocessTweet(tweet); // Process it
+            mc.insertParsedTweet(id,parsedTweet); // Insert it in the DB
+            fc.insert(parsedTweet); // Insert to treemaps
+            if(substancesExist.compareTo("y") == 0) {
+                String substancesString = getSubstances(parsedTweet, tweet);
+                // System.out.println("Substances: " + substancesString + " in tweet: " + parsedTweet);
+                mc.insertSubstancesTwitter(id,substancesString);
+            }
         }
     }
 
+    /**
+     * Checks whether the parsed tweet contains any substances names
+     * @param parsedTweet The parsed tweet to be searched
+     * @param tweet The tweet to be searched
+     * @return A string that contains all the substances mentioned in the tweet
+     */
+    private static String getSubstances(String parsedTweet, String tweet) {
+        /*Optional<String> substances;
+        substances = substancesList.parallelStream().filter(tweet::contains).findAny();
+        if(substances.isPresent()) {
+            System.out.println("SUBSTANCES: " + substances);
+            return substances.get();
+        } else {
+            return "";
+        }*/
+        HashSet<String> subs = new HashSet<>();
+        for(String sub:substancesList) {
+            if(parsedTweet.contains(sub.toLowerCase()) || tweet.contains(sub)) {
+                subs.add(sub);
+            }
+        }
+        return subs.toString();
+    }
+
+
     /**
      * Processes a tweet's text
      * @param input The tweet's text before the processing

src/mongo/MongoConnector.java

@@ -31,6 +31,7 @@ public class MongoConnector {
     private static final String _COMMENT_PARSED_STRING_ = "parsedString";
 
     private static final String EMOTION_SCORES_ = "emScores";
+    private static final String SUBSTANCES_STRING_ = "substancesString";
 
     private static final String _COLL_INDEX_ = "_id";
 
@@ -64,7 +65,7 @@ public void addTweet(String json) throws JSONException {
         JSONObject jsonObj = new JSONObject(json);
         if(_db.getCollection(_coll_name_twitter).find(eq("tweet.id",jsonObj.getString("id"))).first() == null) {
             Document doc = new Document(_TWEET_PARSED_STRING_, null)
-                    .append(_TWEET_JSON_, Document.parse(json));
+                    .append(_TWEET_JSON_, Document.parse(json)).append(SUBSTANCES_STRING_,null);
 
             coll.insertOne(doc);
         } else {
@@ -105,7 +106,9 @@ private boolean insertParsed(ObjectId UUID, String parsed, MongoCollection<Docum
                 return true;
 
             } else {
+                /* Uncomment if you want error messages
                 errorMongo("Replace error -> " + UUID + ", text already parsed!");
+                */
                 return false;
             }
         }
@@ -164,6 +167,15 @@ public HashMap<ObjectId, String> getParsedTweets() {
         return getParsed(parsedTweetsCollection, _TWEET_PARSED_STRING_);
     }
 
+    /**
+     * Gets all substances strings from twitter database
+     * @return Pairs of tweets' IDs and tweets' parsed text
+     */
+    public HashMap<ObjectId, String> getSubstances() {
+        MongoCollection<Document> tweetsCollection = _db.getCollection(_coll_name_twitter);
+        return getParsed(tweetsCollection, SUBSTANCES_STRING_);
+    }
+
     /**
      * Insert emotions to Twitter database
      * @param UUID The Id of the tweet that we want to insert the emotions to.
@@ -217,6 +229,17 @@ private boolean insertEmotions(ObjectId UUID, List<Pair<String/* Emotion Name */
         return false;
     }
 
+    /**
+     * Insert emotions to Twitter database
+     * @param UUID The Id of the tweet that we want to insert the related to.
+     * @param substances A string that contains all the substances that exist in the specific tweet
+     * @return True if the insertion was done correctly, false otherwise
+     */
+    public boolean insertSubstancesTwitter(ObjectId UUID, String substances) {
+        MongoCollection<Document> tweets = _db.getCollection(_coll_name_twitter);
+        return insertParsed(UUID, substances, tweets, SUBSTANCES_STRING_);
+    }
+
     /**
      * Gets all youtube comments from youtube database
      * @return Pairs of comments' IDs and comments' JSONs

src/twitter/main/TwitterExporter.java

@@ -60,6 +60,7 @@ public static void main(String[] args) throws JSONException {
                     criteria.setUntil(parameterSplit[1]);
                 } else if (parameterSplit[0].equals("querysearch")) {
                     //Operators: https://dev.twitter.com/rest/public/search
+                    System.out.println("QUERY: " + parameterSplit[1]);
                     criteria.setQuerySearch(parameterSplit[1]);
                 } else if (parameterSplit[0].equals("maxtweets")) {
                     criteria.setMaxTweets(Integer.valueOf(parameterSplit[1]));

src/twitter/manager/TweetManager.java

@@ -91,11 +91,13 @@ public static List<Tweet> getTweets(TwitterCriteria criteria) {
                 for (Element tweet : tweets) {
                     String usernameTweet = tweet.select("span.username").text();
                     String txt = tweet.select("p.js-tweet-text").text().replaceAll("[^\\u0000-\\uFFFF]", "");
+                    System.out.println("Tweet: " + txt);
                     int retweets = Integer.valueOf(tweet.select("span.ProfileTweet-action--retweet span.ProfileTweet-actionCount").attr("data-tweet-stat-count").replaceAll(",", ""));
                     int favorites = Integer.valueOf(tweet.select("span.ProfileTweet-action--favorite span.ProfileTweet-actionCount").attr("data-tweet-stat-count").replaceAll(",", ""));
                     long dateMs = Long.valueOf(tweet.select("small.time span.js-short-timestamp").attr("data-time-ms"));
                     String id = tweet.attr("data-tweet-id");
                     String permalink = tweet.attr("data-permalink-path");
+                    /* Geo element not working */
                     String geo = "";
                     Elements geoElement = tweet.select("span.Tweet-geo");
                     if (geoElement.size() > 0) {