adding skelliton code

.gitignore

@@ -0,0 +1 @@
+*.pyc

__init__.py

@@ -0,0 +1,6 @@
+__authors__ = ['The Doctor <drwho at virtadpt dot net>','haxwithaxe <me at haxwithaxe dot net>']
+__license__ = "GPLv3"
+
+__help__ = "OH MOD! FIX ME DAMNIT!"
+
+__all__ = ['model', 'view', 'control', 'util']

control/__init__.py

@@ -0,0 +1,13 @@
+"""
+Control (MVC) module
+"""
+
+__authors__ = ["haxwithaxe <me at haxwithaxe dot net>"]
+
+__license__ = "GPLv3"
+
+__help__ = "OH MOD! FIX ME DAMNIT!"
+
+
+# FIXME : Add module names
+__all__ = []

control/db.py

@@ -0,0 +1,50 @@
+"""
+This is the back-end that provides the information data for the UI by gathering collating findings.
+"""
+
+class MissingDB(Exception):
+	pass
+
+# Define Classes
+class UDBI:
+	"""Unified database interface"""
+	def __init__(self):
+		self.db_list = []
+		pass
+
+	def find(self, sources*, **criteria):
+		""" Find database entries from all sources `sources` (if specified otherwise find from any source) with the key-value style criteria specified by `criteria` otherwise find all entries from the specified source.
+		@param	sources		arguments specifying sources to query.
+		@param	criteria	keyword arguments giving criteria for the query
+		@return				dict of results [empty dict if no results]"""
+		pass
+
+	def _db_exist(self, source):
+		"""	See if a required databases exist.
+			@throws		MissingDB exception if a required db is not found
+			@returns	bool
+		"""
+		pass
+
+	def _open_all(self):
+		"""For every directory in database_directory ...
+		model.db.open directory"""
+		# for each source db, append it to self.db_list and open it
+		for db in self.sources:
+			db = DB(db)
+			if db.exists():
+				db.open()
+				self.db_list.append(db)
+
+	def _end_search(self):
+		""" Close the search connections to the databases. """
+		pass
+
+	def _clean_up(self):
+		""" Clean up after ourselves. """
+		pass
+
+	def close(self):
+		""" Close the databases. """
+		pass
+

control/rss.py

@@ -0,0 +1,18 @@
+"""
+RSS Control (MVC) aka Downloader
+"""
+
+
+# not sure if this is still maintained but i recall it was easy to use
+import feedparser
+
+class RSSFeed(URL):
+	"""Sub classes control.URL"""
+	def __init__(self, target, config):
+		pass
+
+	def parse(self):
+		pass
+
+	def dump(self):
+		pass

control/twitter.py

@@ -0,0 +1,72 @@
+This is the module that pulls and indexes a Twitter feed."""
+
+__authors__ = ["The Doctor <drwho at virtadpt dot net>", "haxwithaxe <me at haxwithaxe dot net>"]
+
+__license__ = "GPLv3"
+
+import json
+import twitter
+import ..model.bot.Bot
+
+# Classes
+class TwitterBot(Bot):
+	""" Twwiter scaper Bot. 
+		One feed per Bot """
+
+	def __init__(self, feed_id, config_obj):
+		""" it give it the global Config object or else it gets the hose again ..."""
+		self.conf = config_obj
+		self.db = model.twitter.TwitterDB(self.conf.get("twitter-bot", "db-%s" % feed_id))
+
+	def connect(self):
+		""" Attempt to connect to Twitter's API server. """
+		pass
+
+    def disconnect(self):
+        """ disconnect from Twitter"""
+        pass
+
+	def init_db(self):
+		"""If we had to create the database then we have to load content into it by downloading the whole timeline in chunks and process the information."""
+		pass
+
+	def on_start(self):
+		# Go into a loop in which we wait for X seconds and download new tweets
+		while True:
+			self._update()
+			# After catching up, send the Indexer a call to update.
+			# FIXME: add code to poke indexer
+			# Sleep
+			time.sleep(self.conf.get_int("twitter-bot", "wait-interval", 3600))
+			# Reload the config so config changes can be applied.
+			self.conf.reload()
+
+	def _update(self):
+		""" Download tweets posted since the last recorded tweet in the database. """
+		last_tweet = self._get_last_recorded()
+		# tweets = download from last_tweet['datestamp-field-name'] to now
+		tweets = self._mangle(tweets)
+		self._record(tweets)
+		pass
+
+	def _mangle(self, tweets):
+		""" Format tweets for passing to the database as key value pairs """
+		pass
+
+	def _record(self, tweets):
+		""" Send tweets to the database"""
+		[self.db.set(**tweet) for tweet in tweets]
+
+	def  _get_last_recorded(self):
+		""" Get the datestamp of the last tweet added to the database. """
+		pass
+
+	def clean(self)
+		"""Clean up after ourselves.
+			- Delete tempfiles.
+		"""
+		pass
+
+	def release_db(self):
+		"""Close the databases."""
+		self.db.close()

control/url.py

@@ -0,0 +1,34 @@
+"""
+URL Control (MVC)
+"""
+
+__authors__ = ["haxwithaxe <me at haxwithaxe dot net>"]
+
+__license__ = "GPLv3"
+
+
+class URL:
+	def __init__(self, target):
+		""" config is a model.Config object already loaded
+			@param	target	a string or urllib2.Request object
+		"""
+		self.target = target
+		self.validate_target()	#die hard if we don't have what we need
+
+	def validate_target(self):
+		""" Fail early ... Fail often ... Fail before production
+			@throws		ValueError if self.target is not a string or urllib2.Request
+		"""
+		if not self.target or not isinstance(self.target, str) or not isinstance(self.target, urllib2.Request):
+			raise ValueError("target must be a URI string or a urllib2.Request object.")
+
+	def get(self, ignore_bad_cert=False):
+		""" Get page from target.
+		@param	ignore_bad_cert	bool, if true don't throw erros when ssl cert is invalid/selfsigned
+		@return					the page retrieved or None if not found.
+		"""
+		pass
+
+	def clean(self):
+		""" Clean up files and open sockets or objects. """
+		pass

model/__init__.py

@@ -0,0 +1,13 @@
+"""
+Model (MVC) module
+"""
+
+__authors__ = ["haxwithaxe <me at haxwithaxe dot net>"]
+
+__license__ = "GPLv3"
+
+__help__ = "OH MOD! FIX ME DAMNIT!"
+
+
+# FIXME : Add module names
+__all__ = []

model/bot.py

@@ -0,0 +1,59 @@
+"""
+Bot model
+"""
+
+__authors__ = ["haxwithaxe <me at haxwithaxe dot net>"]
+
+__license__ = "GPLv3"
+
+import pykka
+from  ..util.actor import *
+
+CONNECT = Action("connect")
+DISCONNECT = Action("disconnect")
+
+class Bot(pykka.ThreadingActor):
+	def __init__(self, config):
+		""" config is a model.Config object already loaded"""
+		super(Bot, self).__init__()
+		self.conf = conf
+
+	def on_start(self):
+		""" Hook for doing any setup that should be done after the actor is started, but before it starts processing messages. AKA the main loop """
+		pass
+
+	def on_stop(self):
+		""" Hook for doing any cleanup that should be done after the actor has processed the last message, and before the actor stops. """
+		self.disconnect()
+		self.clean()
+		pass
+
+	def on_failure(self, exception_type, exception_value, traceback):
+		""" Hook for doing any cleanup after an unhandled exception is raised, and before the actor stops. """
+		logging.error(exception_type, exception_value, traceback)
+		pass
+
+	def on_receive(self, message):
+		""" When a message is recieved decide what to do with it
+			@param	message	(picklable dict) – the message to handle
+			@return			anything that should be sent as a reply to the sender
+		"""
+		if message == DISCONNECT:
+			self.disconnect()
+		elif message == CONNECT:
+			self.connect()
+		elif message == CLEAN:
+		else:
+			return self._
+
+	def connect(self):
+		""" Connect to target. """
+		pass
+
+	def disconnect(self):
+		""" Disconnect from target. """
+		pass
+
+	def clean(self):
+		""" Clean up files and open sockets or objects. """
+		pass

model/db.py

@@ -0,0 +1,56 @@
+class Record:
+	""" Database Record model"""
+	default_values = {}
+
+	def __init__(self, **values):
+		self.values = values or {}
+
+	def get(self):
+		""" Get the contents of the record """
+		self._load_defaults()
+		return self.values
+
+	def _load_defaults(self):
+		"""Load the default values so we don't pass any empty fields we don't want to be empty to the database backend"""
+		# shallow copy the default values so we don't mangle it too much
+		defaults = self.default_values.copy()
+		# load the given values over the defaults
+		defaults.update(self.values)
+		# set values to be the updated dictionary
+		self.values = defaults
+
+
+class DB:
+	""" Database model """
+
+	def __init__(self, source):
+		""" Set the database source """
+		pass
+
+	def open(self):
+		""" Open and/or load the database object. """
+		pass
+
+	def get(self, **criteria):
+		""" Find and return the Records that match the criteria. """
+		pass
+
+	def set(self, **records):
+		""" Set the values for the matching Records or create one if there is no match. """
+		pass
+
+	def close(self):
+		""" Close the DB object if required """
+		pass
+
+	def compact(self):
+		"""For every database in database_directory, instantiate a copy of Compactor() and compact the database to free up disk space and make searches more efficient."""
+		pass
+
+	def _file_exists(self):
+		""" Does the database file exist? """
+		pass
+
+	def exists(self):
+		""" Does the database file and/or table exist """
+		pass