Merge pull request #52 from base2Services/develop

Release 0.7.0

README.md

@@ -270,6 +270,10 @@ Available configuration keys are listed below:
 - `shelvery_keep_weekly_backups` - Number of weeks to retain weekly backups
 - `shelvery_keep_monthly_backups` - Number of months to keep monthly backups
 - `shelvery_keep_yearly_backups` - Number of years to keep yearly backups
+
+- `shelvery_custom_retention_types` - custom retention periods in name:seconds (quarterHourly:86400) format, comma separated, empty (disabled) by default
+- `shelvery_current_retention_type` - custom retention period applied to current create backup process
+
 - `shelvery_dr_regions` - List of disaster recovery regions, comma separated
 - `shelvery_wait_snapshot_timeout` - Timeout in seconds to wait for snapshot to become available before copying it
 to another region or sharing with other account. Defaults to 1200 (20 minutes)
@@ -336,6 +340,20 @@ will ensure it's daily backups are retained for 30 days, and copied to `us-west-
 
 Generic format for shelvery config tag is `shevlery:config:$configkey=$configvalue`
 
+### Custom retention periods
+
+Custom retention periods can be set using `shelvery_custom_retention_types` formatted as `[name:retention period]` where retention period defined in seconds.
+Multiple periods can be set using a comma separated list.
+
+```text
+shelvery_custom_retention_types=quarterHourly:86400,hourly:172800
+```
+
+When triggering shelvery on the desired schedule, specify the retention type using `shelvery_current_retention_type` tag with the desired retention type
+
+```text
+shelvery_current_retention_type=quarterHourly
+```
 
 ## Multi account setup
 

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup
 
-setup(name='shelvery', version='0.6.1', author='Base2Services R&D',
+setup(name='shelvery', version='0.7.0', author='Base2Services R&D',
       author_email='[email protected]',
       url='http://github.com/base2Services/shelvery-aws-backups',
       classifiers=[

shelvery/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.6.1'
+__version__ = '0.7.0'
 LAMBDA_WAIT_ITERATION = 'lambda_wait_iteration'
 S3_DATA_PREFIX = 'backups'
 SHELVERY_DO_BACKUP_TAGS = ['True', 'true', '1', 'TRUE']

shelvery/backup_resource.py

@@ -14,26 +14,26 @@
 
 class BackupResource:
     """Model representing single backup"""
-    
+
     BACKUP_MARKER_TAG = 'backup'
     TIMESTAMP_FORMAT = '%Y-%m-%d-%H%M'
     TIMESTAMP_FORMAT_LEGACY = '%Y%m%d-%H%M'
-    
+
     RETENTION_DAILY = 'daily'
     RETENTION_WEEKLY = 'weekly'
     RETENTION_MONTHLY = 'monthly'
     RETENTION_YEARLY = 'yearly'
-    
+
     def __init__(self, tag_prefix, entity_resource: EntityResource, construct=False):
         """Construct new backup resource out of entity resource (e.g. ebs volume)."""
         # if object manually created
         if construct:
             return
-        
+
         # current date
         self.date_created = datetime.utcnow()
         self.account_id = AwsHelper.local_account_id()
-        
+
         # determine retention period
         if self.date_created.day == 1:
             if self.date_created.month == 1:
@@ -44,15 +44,15 @@ def __init__(self, tag_prefix, entity_resource: EntityResource, construct=False)
             self.retention_type = self.RETENTION_WEEKLY
         else:
             self.retention_type = self.RETENTION_DAILY
-        
+
         # determine backup name. Hash of resource id is added to support creating backups
         # with resources having a same name
         if 'Name' in entity_resource.tags:
             name = entity_resource.tags['Name']
             name = name + '-' + hashlib.md5(entity_resource.resource_id.encode('utf-8')).hexdigest()[0:6]
         else:
             name = entity_resource.resource_id
-        
+
         # replace anything that is not alphanumeric to hyphen
         # do not allow two hyphens next to each other
         name = re.sub('[^a-zA-Z0-9\-]', '-', name)
@@ -63,7 +63,7 @@ def __init__(self, tag_prefix, entity_resource: EntityResource, construct=False)
         self.entity_id = entity_resource.resource_id
         self.entity_resource = entity_resource
         self.__region = entity_resource.resource_region
-    
+
         self.tags = {
             'Name': self.name,
             "shelvery:tag_name": tag_prefix,
@@ -78,15 +78,15 @@ def __init__(self, tag_prefix, entity_resource: EntityResource, construct=False)
         self.backup_id = None
         self.expire_date = None
         self.date_deleted = None
-        
+
     def cross_account_copy(self, new_backup_id):
         backup = copy.deepcopy(self)
 
         # backup name and retention type are copied
         backup.backup_id = new_backup_id
         backup.region = AwsHelper.local_region()
         backup.account_id = AwsHelper.local_account_id()
-        
+
         tag_prefix = self.tags['shelvery:tag_name']
         backup.tags[f"{tag_prefix}:region"] = backup.region
         backup.tags[f"{tag_prefix}:date_copied"] =  datetime.utcnow().strftime(self.TIMESTAMP_FORMAT)
@@ -98,8 +98,8 @@ def cross_account_copy(self, new_backup_id):
         backup.tags[f"{tag_prefix}:dr_regions"] = ''
         backup.tags[f"{tag_prefix}:dr_copies"] = ''
         return backup
-    
-    
+
+
     @classmethod
     def construct(cls,
                   tag_prefix: str,
@@ -108,41 +108,41 @@ def construct(cls,
         """
         Construct BackupResource object from object id and aws tags stored by shelvery
         """
-        
+
         obj = BackupResource(None, None, True)
         obj.entity_resource = None
         obj.entity_id = None
         obj.backup_id = backup_id
         obj.tags = tags
-        
+
         # read properties from tags
         obj.retention_type = tags[f"{tag_prefix}:retention_type"]
         obj.name = tags[f"{tag_prefix}:name"]
-        
+
         if f"{tag_prefix}:entity_id" in tags:
             obj.entity_id = tags[f"{tag_prefix}:entity_id"]
-            
+
         try:
             obj.date_created = datetime.strptime(tags[f"{tag_prefix}:date_created"], cls.TIMESTAMP_FORMAT)
         except Exception as e:
             if 'does not match format' in str(e):
                 str_date = tags[f"{tag_prefix}:date_created"]
                 print(f"Failed to read {str_date} as date, trying legacy format {cls.TIMESTAMP_FORMAT_LEGACY}")
                 obj.date_created = datetime.strptime(tags[f"{tag_prefix}:date_created"], cls.TIMESTAMP_FORMAT_LEGACY)
-                
-                
+
+
         obj.region = tags[f"{tag_prefix}:region"]
         if f"{tag_prefix}:src_account" in tags:
             obj.account_id = tags[f"{tag_prefix}:src_account"]
         else:
             obj.account_id = AwsHelper.local_account_id()
-            
+
         return obj
 
     def entity_resource_tags(self):
         return self.entity_resource.tags if self.entity_resource is not None else {}
 
-    def calculate_expire_date(self, engine):
+    def calculate_expire_date(self, engine, custom_retention_types=None):
         """Determine expire date, based on 'retention_type' tag"""
         if self.retention_type == BackupResource.RETENTION_DAILY:
             expire_date = self.date_created + timedelta(
@@ -156,30 +156,39 @@ def calculate_expire_date(self, engine):
         elif self.retention_type == BackupResource.RETENTION_YEARLY:
             expire_date = self.date_created + relativedelta(
                 years=RuntimeConfig.get_keep_yearly(self.entity_resource_tags(), engine))
+        elif self.retention_type in custom_retention_types:
+            expire_date =  self.date_created + timedelta(
+                seconds=custom_retention_types[self.retention_type])
         else:
             # in case there is no retention tag on backup, we want it kept forever
             expire_date = datetime.utcnow() + relativedelta(years=10)
-        
+
         self.expire_date = expire_date
-    
-    def is_stale(self, engine):
-        self.calculate_expire_date(engine)
+
+    def is_stale(self, engine, custom_retention_types = None):
+        self.calculate_expire_date(engine, custom_retention_types)
         now = datetime.now(self.date_created.tzinfo)
         return now > self.expire_date
-    
+
     @property
     def region(self):
         return self.__region
-    
+
     @region.setter
     def region(self, region: str):
         self.__region = region
 
+    def set_retention_type(self, retention_type: str):
+        self.name = '-'.join(self.name.split('-')[0:-1]) + f"-{retention_type}"
+        self.tags[f"{self.tags['shelvery:tag_name']}:name"] = self.name
+        self.tags['Name'] = self.name
+        self.tags[f"{self.tags['shelvery:tag_name']}:retention_type"] = retention_type
+
     @property
     def boto3_tags(self):
         tags = self.tags
         return list(map(lambda k: {'Key': k, 'Value': tags[k]}, tags))
-    
+
     @staticmethod
     def dict_from_boto3_tags(boot3_tags):
-        return dict(map(lambda t: (t['Key'], t['Value']), boot3_tags))
+        return dict(map(lambda t: (t['Key'], t['Value']), boot3_tags))

shelvery/engine.py

@@ -167,11 +167,16 @@ def create_backups(self) -> List[BackupResource]:
 
         # create and collect backups
         backup_resources = []
+        current_retention_type = RuntimeConfig.get_current_retention_type(self)
         for r in resources:
             backup_resource = BackupResource(
                 tag_prefix=RuntimeConfig.get_tag_prefix(),
                 entity_resource=r
             )
+            # if retention is explicitly given by runtime environment
+            if current_retention_type is not None:
+                backup_resource.set_retention_type(current_retention_type)
+
             dr_regions = RuntimeConfig.get_dr_regions(backup_resource.entity_resource.tags, self)
             backup_resource.tags[f"{RuntimeConfig.get_tag_prefix()}:dr_regions"] = ','.join(dr_regions)
             self.logger.info(f"Processing {resource_type} with id {r.resource_id}")
@@ -236,7 +241,7 @@ def clean_backups(self):
         for backup in existing_backups:
             self.logger.info(f"Checking backup {backup.backup_id}")
             try:
-                if backup.is_stale(self):
+                if backup.is_stale(self, RuntimeConfig.get_custom_retention_types(self)):
                     self.logger.info(
                         f"{backup.retention_type} backup {backup.name} has expired on {backup.expire_date}, cleaning up")
                     self.delete_backup(backup)

shelvery/runtime_config.py

@@ -13,6 +13,8 @@ class RuntimeConfig:
     shelvery_keep_monthly_backups - daily backups to keep, defaults to 12 months
     shelvery_keep_yearly_backups - daily backups to keep, defaults to 10 years
 
+    shelvery_custom_retention_types - custom retention periods in name:seconds format, comma separated, empty (disabled) by default
+    shelvery_current_retention_type - custom retention period applied to current create backup process
     shelvery_dr_regions - disaster recovery regions, comma separated, empty (disabled) by default
 
     shelvery_keep_daily_backups_dr - daily backups to keep in disaster recover region
@@ -58,6 +60,8 @@ class RuntimeConfig:
         'shelvery_keep_weekly_backups': 8,
         'shelvery_keep_monthly_backups': 12,
         'shelvery_keep_yearly_backups': 10,
+        'shelvery_custom_retention_types': None,
+        'shelvery_current_retention_type': None,
         'shelvery_wait_snapshot_timeout': 1200,
         'shelvery_lambda_max_wait_iterations': 5,
         'shelvery_dr_regions': None,
@@ -108,6 +112,27 @@ def get_keep_monthly(cls, resource_tags=None, engine=None):
     def get_keep_yearly(cls, resource_tags=None, engine=None):
         return int(cls.get_conf_value('shelvery_keep_yearly_backups', resource_tags, engine.lambda_payload))
 
+    @classmethod
+    def get_custom_retention_types(cls, engine=None):
+        custom_retention = cls.get_conf_value('shelvery_custom_retention_types', None, engine.lambda_payload)
+        if custom_retention is None or custom_retention.strip() == '':
+            return {}
+
+        retentions = custom_retention.split(',')
+        rval = {}
+        for retention in retentions:
+            parts = retention.split(':')
+            if len(parts) == 2:
+                rval[parts[0]] = int(parts[1])
+        return rval
+
+    @classmethod
+    def get_current_retention_type(cls, engine=None):
+        current_retention_type = cls.get_conf_value('shelvery_current_retention_type', None, engine.lambda_payload)
+        if current_retention_type is None or current_retention_type.strip() == '':
+            return None
+        return current_retention_type
+
     @classmethod
     def get_envvalue(cls, key: str, default_value):
         return os.environ[key] if key in os.environ else default_value