Fix for twitter client double polling issue. Updated Weather grammars

.gitignore

@@ -103,14 +103,15 @@ license.keys
 bots/y-bot/output/
 /bots/alice2/output/
 /bots/professor/output/
-/scratch/
 temp/
 conversations/
 /src/utils/pattern_lister/y-bot.csv
 /src/utils/rdf_lister/rdfs.csv
 
 /bots/scratch/
 bots/scratch/
+/scratch/
+/src/scratch
 
 /bots/multibot/
 bots/multibot/

src/programy/bot.py

@@ -206,9 +206,10 @@ def initiate_conversation_storage(self):
 
     def load_conversation(self, clientid):
         if self._conversation_storage is not None:
-            conversation = self._conversations[clientid]
-            self._conversation_storage.load_conversation(conversation, clientid,
-                                                         self.configuration.conversations.restore_last_topic)
+            if clientid in self._conversations:
+                conversation = self._conversations[clientid]
+                self._conversation_storage.load_conversation(conversation, clientid,
+                                                             self.configuration.conversations.restore_last_topic)
 
     def save_conversation(self, clientid):
         if self._conversation_storage is not None:

src/programy/clients/twitter.py

@@ -26,6 +26,8 @@
 
 class TwitterBotClient(BotClient):
 
+    FIFTEEN_MINUTES = 15*60
+
     def __init__(self, argument_parser=None):
         self._username = "unknown"
         self._username_len = 0
@@ -150,7 +152,9 @@ def _get_statuses(self, last_status_id):
         else:
             if logging.getLogger().isEnabledFor(logging.DEBUG):
                 logging.debug("Getting latest statuses since : %s", last_status_id)
+
             statuses = self._api.home_timeline(since_id=last_status_id)
+
         statuses.sort(key=lambda msg: msg.id)
         return statuses
 
@@ -183,6 +187,10 @@ def _process_status_question(self, userid, text):
 
             if logging.getLogger().isEnabledFor(logging.DEBUG):
                 logging.debug(status)
+
+            if logging.getLogger().isEnabledFor(logging.DEBUG):
+                logging.debug("Sending status response [@%s] [%s]", (user.screen_name, response))
+
             self._api.update_status(status)
 
     def _process_statuses(self, last_status_id):
@@ -192,7 +200,10 @@ def _process_statuses(self, last_status_id):
         statuses = self._get_statuses(last_status_id)
 
         for status in statuses:
-            #print("[%s] - [%s]"%(status.author.screen_name, self._username))
+
+            if logging.getLogger().isEnabledFor(logging.DEBUG):
+                logging.debug("%s Received Status From[%s] - To[%s] -> [%s]", status.id, status.author.screen_name, self._username, status.text)
+
             if status.author.screen_name != self._username:
                 if logging.getLogger().isEnabledFor(logging.DEBUG):
                     logging.debug("status: %s", status.text)
@@ -202,8 +213,7 @@ def _process_statuses(self, last_status_id):
                     if logging.getLogger().isEnabledFor(logging.ERROR):
                         logging.error(err)
 
-        if statuses:
-            last_status_id = statuses[-1].id
+            last_status_id = status.id
 
         return last_status_id
 
@@ -225,16 +235,26 @@ def _get_last_message_ids(self):
                 except Exception as excep:
                     logging.exception(excep)
 
+        if logging.getLogger().isEnabledFor(logging.DEBUG):
+            logging.debug ("Got Last Messaged ID: %s", last_direct_message_id)
+        if logging.getLogger().isEnabledFor(logging.DEBUG):
+            logging.debug ("Got Last Status ID: %s", last_status_id)
+
         return (last_direct_message_id, last_status_id)
 
     def _store_last_message_ids(self, last_direct_message_id, last_status_id):
         if self.configuration.client_configuration.storage == 'file':
-            if logging.getLogger().isEnabledFor(logging.DEBUG):
-                logging.debug("Writing messages ids to [%s]", self.configuration.client_configuration.storage_location)
             try:
+                if logging.getLogger().isEnabledFor(logging.DEBUG):
+                    logging.debug("Storing Last Messaged ID: %s", last_direct_message_id)
+
+                if logging.getLogger().isEnabledFor(logging.DEBUG):
+                    logging.debug("Storing Last Status ID: %s", last_status_id)
+
                 with open(self.configuration.client_configuration.storage_location, "w+", encoding="utf-8") as idfile:
                     idfile.write("%d\n"%last_direct_message_id)
                     idfile.write("%d\n"%last_status_id)
+
             except Exception as excep:
                 logging.exception(excep)
 
@@ -246,12 +266,21 @@ def _poll(self, last_direct_message_id, last_status_id):
             if self.configuration.client_configuration.auto_follow is True:
                 self._process_followers()
 
+            if logging.getLogger().isEnabledFor(logging.DEBUG):
+                logging.debug("Processing direct messaages")
+
             last_direct_message_id = self._process_direct_messages(last_direct_message_id)
+
             if logging.getLogger().isEnabledFor(logging.DEBUG):
                 logging.debug("Last message id = %d", last_direct_message_id)
 
         if self.configuration.client_configuration.use_status is True:
+
+            if logging.getLogger().isEnabledFor(logging.DEBUG):
+                logging.debug("Processing status messaages")
+
             last_status_id = self._process_statuses(last_status_id)
+
             if logging.getLogger().isEnabledFor(logging.DEBUG):
                 logging.debug("Last status id = %d", last_status_id)
 
@@ -263,23 +292,27 @@ def _use_polling(self):
 
         print("Twitter client running as [%s]..."%self._username)
 
-        (last_direct_message_id, last_status_id) = self._get_last_message_ids()
-
         running = True
         while running is True:
             try:
+                (last_direct_message_id, last_status_id) = self._get_last_message_ids()
+
                 self._poll(last_direct_message_id, last_status_id)
 
             except KeyboardInterrupt:
                 running = False
 
             except RateLimitError:
-                if logging.getLogger().isEnabledFor(logging.ERROR):
-                    logging.error("Rate limit exceeded, sleeping for 15 minutes")
-                if self.configuration.client_configuration.poll_sleep != -1:
-                    time.sleep(self.configuration.client_configuration.rate_limit_sleep)
+
+                if self.configuration.client_configuration.rate_limit_sleep != -1:
+                    rate_limit_sleep = self.configuration.client_configuration.rate_limit_sleep
                 else:
-                    time.sleep(15*60)
+                    rate_limit_sleep = self.FIFTEEN_MINUTES
+
+                if logging.getLogger().isEnabledFor(logging.ERROR):
+                    logging.error("Rate limit exceeded, sleeping for %d seconds", rate_limit_sleep)
+
+                time.sleep(rate_limit_sleep)
 
             except Exception as excep:
                 logging.exception(excep)

src/programy/extensions/geocode/geocode.py

@@ -22,6 +22,9 @@
 
 class GeoCodeExtension(Extension):
 
+    def get_geo_locator(self):
+        return  GoogleMaps()
+
     # execute() is the interface that is called from the <extension> tag in the AIML
     def execute(self, bot, clientid, data):
         if logging.getLogger().isEnabledFor(logging.DEBUG):
@@ -37,7 +40,8 @@ def execute(self, bot, clientid, data):
         else:
             return None
 
-        googlemaps = GoogleMaps(bot.license_keys)
+        googlemaps = self.get_geo_locator()
+
         latlng = googlemaps.get_latlong_for_location(location)
         if latlng is not None:
             str_lat = str(latlng.latitude)

src/programy/extensions/weather/weather.py

@@ -15,6 +15,7 @@
 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 """
 import logging
+import datetime
 
 from programy.utils.weather.metoffice import MetOffice
 from programy.utils.geo.google import GoogleMaps
@@ -23,6 +24,12 @@
 
 class WeatherExtension(Extension):
 
+    def get_geo_locator(self, bot):
+        return GoogleMaps()
+
+    def get_met_office(self, bot):
+        return MetOffice(bot.license_keys)
+
     # WEATHER [OBSERVATION|FORECAST3|FORECAST24] LOCATION * WHEN *
 
     # execute() is the interface that is called from the <extension> tag in the AIML
@@ -35,7 +42,7 @@ def execute(self, bot, clientid, data):
             return None
 
         type = splits[0]
-        if type not in ['OBSERVATION', 'FORECAST3', 'FORECAST24']:
+        if type not in ['OBSERVATION', 'FORECAST5DAY', 'FORECAST24HOUR']:
             if logging.getLogger().isEnabledFor(logging.DEBUG):
                 logging.debug("Weather - Type not understood [%s]"%type)
             return None
@@ -55,54 +62,61 @@ def execute(self, bot, clientid, data):
             return None
 
         if type == 'OBSERVATION':
-            return self.get_observation(bot, clientid, postcode, when)
-        elif type == 'FORECAST3':
-            return self.get_forecast3(bot, clientid, postcode, when)
-        elif type == 'FORECAST24':
-            return self.get_forecast24(bot, clientid, postcode, when)
+            return self.current_observation(bot, postcode)
+        elif type == 'FORECAST5DAY':
+            return self.five_day_forecast(bot, postcode, when)
+        elif type == 'FORECAST24HOUR':
+            return self.twentyfour_hour_forecast(bot, postcode, when)
 
-    def get_observation(self, bot, clientid, postcode, when):
+    def current_observation(self, bot, postcode):
         if logging.getLogger().isEnabledFor(logging.DEBUG):
-            logging.debug("Getting weather observation for [%s] at time [%s]"%(postcode, when))
+            logging.debug("Getting weather observation for [%s]"%(postcode))
 
-        googlemaps = GoogleMaps(bot.license_keys)
+        googlemaps = self.get_geo_locator(bot)
         latlng = googlemaps.get_latlong_for_location(postcode)
 
-        met_office = MetOffice(bot.license_keys)
+        met_office = self.get_met_office(bot)
 
         observation = met_office.current_observation(latlng.latitude, latlng.longitude)
         if observation is not None:
-            return observation.get_latest().to_program_y_text()
+            #TODO Use 'when" paramter to extract datapoints
+            return observation.get_latest_observation().to_program_y_text()
         else:
-            return None
+            return "UNAVAILABLE"
 
-    def get_forecast3(self, bot, clientid, postcode, when):
+    def twentyfour_hour_forecast(self, bot, postcode, when):
         if logging.getLogger().isEnabledFor(logging.DEBUG):
-            logging.debug("Getting 3 hourly weather forecast for [%s] at time [%s]"%(postcode, when))
+            logging.debug("Getting 24 hour weather forecast for [%s] at time [%s]"%(postcode, when))
 
-        googlemaps = GoogleMaps(bot.license_keys)
+        googlemaps = self.get_geo_locator(bot)
         latlng = googlemaps.get_latlong_for_location(postcode)
 
-        met_office = MetOffice(bot.license_keys)
+        met_office = self.get_met_office(bot)
 
-        forecast = met_office.three_hourly_forecast(latlng.latitude, latlng.longitude)
+        forecast = met_office.twentyfour_hour_forecast(latlng.latitude, latlng.longitude)
         if forecast is not None:
-            return forecast.get_latest().to_program_y_text()
+            datapoint = forecast.get_forecast_for_n_hours_ahead(int(when))
+            if datapoint is None:
+                datapoint = forecast.get_latest_forecast().to_program_y_text()
+            return datapoint
         else:
-            return None
+            return "UNAVAILABLE"
 
-    def get_forecast24(self, bot, clientid, postcode, when):
+    def five_day_forecast(self, bot, postcode, when):
         if logging.getLogger().isEnabledFor(logging.DEBUG):
-            logging.debug("Getting 24 hour weather forecast for [%s] at time [%s]"%(postcode, when))
+            logging.debug("Getting 5 day forecast for [%s] at time [%s]"%(postcode, when))
 
-        googlemaps = GoogleMaps(bot.license_keys)
+        googlemaps = self.get_geo_locator(bot)
         latlng = googlemaps.get_latlong_for_location(postcode)
 
-        met_office = MetOffice(bot.license_keys)
+        met_office = self.get_met_office(bot)
 
-        forecast = met_office.daily_forecast(latlng.latitude, latlng.longitude)
+        forecast = met_office.five_day_forecast(latlng.latitude, latlng.longitude)
         if forecast is not None:
-            return forecast.get_latest().to_program_y_text()
+            datapoint = forecast.get_forecast_for_n_days_ahead(int(when))
+            if datapoint is None:
+                datapoint = forecast.get_latest_forecast().to_program_y_text()
+            return datapoint
         else:
-            return None
+            return "UNAVAILABLE"
 

src/programy/parser/factory.py

@@ -52,13 +52,9 @@ def process_config_line(self, line):
     def valid_config_line(self, line):
 
         if not line:
-            if logging.getLogger().isEnabledFor(logging.ERROR):
-                logging.error("Config line empty")
             return False
 
         if line.startswith('#'):
-            if logging.getLogger().isEnabledFor(logging.ERROR):
-                logging.error("Config line is comment")
             return False
 
         if "=" not in line:

src/programy/parser/template/nodes/get.py

@@ -58,8 +58,8 @@ def tuples(self, tuples):
     def get_default_value(bot):
         value = bot.brain.properties.property("default-get")
         if value is None:
-            if logging.getLogger().isEnabledFor(logging.ERROR):
-                logging.error("No property defined for default-get")
+            if logging.getLogger().isEnabledFor(logging.INFO):
+                logging.error("No property defined for default-get, checking defaults")
 
             value = bot.brain.configuration.defaults.default_get
             if value is None: