diff --git a/README.md b/README.md index c8ac686..c8e4a9f 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,10 @@ Available configuration keys are listed below: - `shelvery_keep_weekly_backups` - Number of weeks to retain weekly backups - `shelvery_keep_monthly_backups` - Number of months to keep monthly backups - `shelvery_keep_yearly_backups` - Number of years to keep yearly backups + +- `shelvery_custom_retention_types` - custom retention periods in name:seconds (quarterHourly:86400) format, comma separated, empty (disabled) by default +- `shelvery_current_retention_type` - custom retention period applied to current create backup process + - `shelvery_dr_regions` - List of disaster recovery regions, comma separated - `shelvery_wait_snapshot_timeout` - Timeout in seconds to wait for snapshot to become available before copying it to another region or sharing with other account. Defaults to 1200 (20 minutes) @@ -336,6 +340,20 @@ will ensure it's daily backups are retained for 30 days, and copied to `us-west- Generic format for shelvery config tag is `shevlery:config:$configkey=$configvalue` +### Custom retention periods + +Custom retention periods can be set using `shelvery_custom_retention_types` formatted as `[name:retention period]` where retention period defined in seconds. +Multiple periods can be set using a comma separated list. + +```text +shelvery_custom_retention_types=quarterHourly:86400,hourly:172800 +``` + +When triggering shelvery on the desired schedule, specify the retention type using `shelvery_current_retention_type` tag with the desired retention type + +```text +shelvery_current_retention_type=quarterHourly +``` ## Multi account setup diff --git a/setup.py b/setup.py index 023f109..0d2c31f 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup -setup(name='shelvery', version='0.6.1', author='Base2Services R&D', +setup(name='shelvery', version='0.7.0', author='Base2Services R&D', author_email='itsupport@base2services.com', url='http://github.com/base2Services/shelvery-aws-backups', classifiers=[ diff --git a/shelvery/__init__.py b/shelvery/__init__.py index b89cb01..8dfd74f 100644 --- a/shelvery/__init__.py +++ b/shelvery/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.6.1' +__version__ = '0.7.0' LAMBDA_WAIT_ITERATION = 'lambda_wait_iteration' S3_DATA_PREFIX = 'backups' SHELVERY_DO_BACKUP_TAGS = ['True', 'true', '1', 'TRUE'] diff --git a/shelvery/backup_resource.py b/shelvery/backup_resource.py index 2710867..6cb0729 100644 --- a/shelvery/backup_resource.py +++ b/shelvery/backup_resource.py @@ -14,26 +14,26 @@ class BackupResource: """Model representing single backup""" - + BACKUP_MARKER_TAG = 'backup' TIMESTAMP_FORMAT = '%Y-%m-%d-%H%M' TIMESTAMP_FORMAT_LEGACY = '%Y%m%d-%H%M' - + RETENTION_DAILY = 'daily' RETENTION_WEEKLY = 'weekly' RETENTION_MONTHLY = 'monthly' RETENTION_YEARLY = 'yearly' - + def __init__(self, tag_prefix, entity_resource: EntityResource, construct=False): """Construct new backup resource out of entity resource (e.g. ebs volume).""" # if object manually created if construct: return - + # current date self.date_created = datetime.utcnow() self.account_id = AwsHelper.local_account_id() - + # determine retention period if self.date_created.day == 1: if self.date_created.month == 1: @@ -44,7 +44,7 @@ def __init__(self, tag_prefix, entity_resource: EntityResource, construct=False) self.retention_type = self.RETENTION_WEEKLY else: self.retention_type = self.RETENTION_DAILY - + # determine backup name. Hash of resource id is added to support creating backups # with resources having a same name if 'Name' in entity_resource.tags: @@ -52,7 +52,7 @@ def __init__(self, tag_prefix, entity_resource: EntityResource, construct=False) name = name + '-' + hashlib.md5(entity_resource.resource_id.encode('utf-8')).hexdigest()[0:6] else: name = entity_resource.resource_id - + # replace anything that is not alphanumeric to hyphen # do not allow two hyphens next to each other name = re.sub('[^a-zA-Z0-9\-]', '-', name) @@ -63,7 +63,7 @@ def __init__(self, tag_prefix, entity_resource: EntityResource, construct=False) self.entity_id = entity_resource.resource_id self.entity_resource = entity_resource self.__region = entity_resource.resource_region - + self.tags = { 'Name': self.name, "shelvery:tag_name": tag_prefix, @@ -78,7 +78,7 @@ def __init__(self, tag_prefix, entity_resource: EntityResource, construct=False) self.backup_id = None self.expire_date = None self.date_deleted = None - + def cross_account_copy(self, new_backup_id): backup = copy.deepcopy(self) @@ -86,7 +86,7 @@ def cross_account_copy(self, new_backup_id): backup.backup_id = new_backup_id backup.region = AwsHelper.local_region() backup.account_id = AwsHelper.local_account_id() - + tag_prefix = self.tags['shelvery:tag_name'] backup.tags[f"{tag_prefix}:region"] = backup.region backup.tags[f"{tag_prefix}:date_copied"] = datetime.utcnow().strftime(self.TIMESTAMP_FORMAT) @@ -98,8 +98,8 @@ def cross_account_copy(self, new_backup_id): backup.tags[f"{tag_prefix}:dr_regions"] = '' backup.tags[f"{tag_prefix}:dr_copies"] = '' return backup - - + + @classmethod def construct(cls, tag_prefix: str, @@ -108,20 +108,20 @@ def construct(cls, """ Construct BackupResource object from object id and aws tags stored by shelvery """ - + obj = BackupResource(None, None, True) obj.entity_resource = None obj.entity_id = None obj.backup_id = backup_id obj.tags = tags - + # read properties from tags obj.retention_type = tags[f"{tag_prefix}:retention_type"] obj.name = tags[f"{tag_prefix}:name"] - + if f"{tag_prefix}:entity_id" in tags: obj.entity_id = tags[f"{tag_prefix}:entity_id"] - + try: obj.date_created = datetime.strptime(tags[f"{tag_prefix}:date_created"], cls.TIMESTAMP_FORMAT) except Exception as e: @@ -129,20 +129,20 @@ def construct(cls, str_date = tags[f"{tag_prefix}:date_created"] print(f"Failed to read {str_date} as date, trying legacy format {cls.TIMESTAMP_FORMAT_LEGACY}") obj.date_created = datetime.strptime(tags[f"{tag_prefix}:date_created"], cls.TIMESTAMP_FORMAT_LEGACY) - - + + obj.region = tags[f"{tag_prefix}:region"] if f"{tag_prefix}:src_account" in tags: obj.account_id = tags[f"{tag_prefix}:src_account"] else: obj.account_id = AwsHelper.local_account_id() - + return obj def entity_resource_tags(self): return self.entity_resource.tags if self.entity_resource is not None else {} - def calculate_expire_date(self, engine): + def calculate_expire_date(self, engine, custom_retention_types=None): """Determine expire date, based on 'retention_type' tag""" if self.retention_type == BackupResource.RETENTION_DAILY: expire_date = self.date_created + timedelta( @@ -156,30 +156,39 @@ def calculate_expire_date(self, engine): elif self.retention_type == BackupResource.RETENTION_YEARLY: expire_date = self.date_created + relativedelta( years=RuntimeConfig.get_keep_yearly(self.entity_resource_tags(), engine)) + elif self.retention_type in custom_retention_types: + expire_date = self.date_created + timedelta( + seconds=custom_retention_types[self.retention_type]) else: # in case there is no retention tag on backup, we want it kept forever expire_date = datetime.utcnow() + relativedelta(years=10) - + self.expire_date = expire_date - - def is_stale(self, engine): - self.calculate_expire_date(engine) + + def is_stale(self, engine, custom_retention_types = None): + self.calculate_expire_date(engine, custom_retention_types) now = datetime.now(self.date_created.tzinfo) return now > self.expire_date - + @property def region(self): return self.__region - + @region.setter def region(self, region: str): self.__region = region + def set_retention_type(self, retention_type: str): + self.name = '-'.join(self.name.split('-')[0:-1]) + f"-{retention_type}" + self.tags[f"{self.tags['shelvery:tag_name']}:name"] = self.name + self.tags['Name'] = self.name + self.tags[f"{self.tags['shelvery:tag_name']}:retention_type"] = retention_type + @property def boto3_tags(self): tags = self.tags return list(map(lambda k: {'Key': k, 'Value': tags[k]}, tags)) - + @staticmethod def dict_from_boto3_tags(boot3_tags): - return dict(map(lambda t: (t['Key'], t['Value']), boot3_tags)) \ No newline at end of file + return dict(map(lambda t: (t['Key'], t['Value']), boot3_tags)) diff --git a/shelvery/engine.py b/shelvery/engine.py index e86fdda..800585b 100644 --- a/shelvery/engine.py +++ b/shelvery/engine.py @@ -167,11 +167,16 @@ def create_backups(self) -> List[BackupResource]: # create and collect backups backup_resources = [] + current_retention_type = RuntimeConfig.get_current_retention_type(self) for r in resources: backup_resource = BackupResource( tag_prefix=RuntimeConfig.get_tag_prefix(), entity_resource=r ) + # if retention is explicitly given by runtime environment + if current_retention_type is not None: + backup_resource.set_retention_type(current_retention_type) + dr_regions = RuntimeConfig.get_dr_regions(backup_resource.entity_resource.tags, self) backup_resource.tags[f"{RuntimeConfig.get_tag_prefix()}:dr_regions"] = ','.join(dr_regions) self.logger.info(f"Processing {resource_type} with id {r.resource_id}") @@ -236,7 +241,7 @@ def clean_backups(self): for backup in existing_backups: self.logger.info(f"Checking backup {backup.backup_id}") try: - if backup.is_stale(self): + if backup.is_stale(self, RuntimeConfig.get_custom_retention_types(self)): self.logger.info( f"{backup.retention_type} backup {backup.name} has expired on {backup.expire_date}, cleaning up") self.delete_backup(backup) diff --git a/shelvery/runtime_config.py b/shelvery/runtime_config.py index cb36046..d92c9cb 100644 --- a/shelvery/runtime_config.py +++ b/shelvery/runtime_config.py @@ -13,6 +13,8 @@ class RuntimeConfig: shelvery_keep_monthly_backups - daily backups to keep, defaults to 12 months shelvery_keep_yearly_backups - daily backups to keep, defaults to 10 years + shelvery_custom_retention_types - custom retention periods in name:seconds format, comma separated, empty (disabled) by default + shelvery_current_retention_type - custom retention period applied to current create backup process shelvery_dr_regions - disaster recovery regions, comma separated, empty (disabled) by default shelvery_keep_daily_backups_dr - daily backups to keep in disaster recover region @@ -58,6 +60,8 @@ class RuntimeConfig: 'shelvery_keep_weekly_backups': 8, 'shelvery_keep_monthly_backups': 12, 'shelvery_keep_yearly_backups': 10, + 'shelvery_custom_retention_types': None, + 'shelvery_current_retention_type': None, 'shelvery_wait_snapshot_timeout': 1200, 'shelvery_lambda_max_wait_iterations': 5, 'shelvery_dr_regions': None, @@ -108,6 +112,27 @@ def get_keep_monthly(cls, resource_tags=None, engine=None): def get_keep_yearly(cls, resource_tags=None, engine=None): return int(cls.get_conf_value('shelvery_keep_yearly_backups', resource_tags, engine.lambda_payload)) + @classmethod + def get_custom_retention_types(cls, engine=None): + custom_retention = cls.get_conf_value('shelvery_custom_retention_types', None, engine.lambda_payload) + if custom_retention is None or custom_retention.strip() == '': + return {} + + retentions = custom_retention.split(',') + rval = {} + for retention in retentions: + parts = retention.split(':') + if len(parts) == 2: + rval[parts[0]] = int(parts[1]) + return rval + + @classmethod + def get_current_retention_type(cls, engine=None): + current_retention_type = cls.get_conf_value('shelvery_current_retention_type', None, engine.lambda_payload) + if current_retention_type is None or current_retention_type.strip() == '': + return None + return current_retention_type + @classmethod def get_envvalue(cls, key: str, default_value): return os.environ[key] if key in os.environ else default_value