From ffbb9bfa6592ad22b18b308bdcac4ab9dd306465 Mon Sep 17 00:00:00 2001 From: Timothy Castillo Date: Thu, 11 Jul 2019 21:56:53 -0500 Subject: [PATCH] Update to use crash-consitent snapshots --- ebs_snapper/snapshot.py | 20 ++++++++++++-------- ebs_snapper/utils.py | 42 +++++++++++++++++++---------------------- requirements.txt | 4 ++-- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/ebs_snapper/snapshot.py b/ebs_snapper/snapshot.py index 797e6ea..6bd326b 100644 --- a/ebs_snapper/snapshot.py +++ b/ebs_snapper/snapshot.py @@ -126,6 +126,11 @@ def perform_snapshot(context, region, installed_region='us-east-1'): ami_id = instance_data['ImageId'] LOG.info('Reviewing snapshots in region %s on instance %s', region, instance_id) + # grabbing nuber of volumes attached. If there is a volume that is atttached + # which has at least one snapshot due, it will call the function + dev = instance_data.get('BlockDeviceMappings', []) + dev_due_count = 0 + for dev in instance_data.get('BlockDeviceMappings', []): # before we go make a bunch more API calls if timeout_check(context, 'perform_snapshot'): @@ -136,9 +141,6 @@ def perform_snapshot(context, region, installed_region='us-east-1'): LOG.debug('Considering device %s', dev) volume_id = dev['Ebs']['VolumeId'] - if volume_id in ignore_ids: - continue - # find snapshots recent = volume_snap_recent.get(volume_id) now = datetime.datetime.now(dateutil.tz.tzutc()) @@ -146,26 +148,28 @@ def perform_snapshot(context, region, installed_region='us-east-1'): # snapshot due? if should_perform_snapshot(frequency, now, volume_id, recent): LOG.debug('Performing snapshot for %s, calculating tags', volume_id) + dev_due_count = dev_due_count +1 else: - LOG.debug('NOT Performing snapshot for %s', volume_id) + LOG.debug('Snapshot for %s is not due at this time', volume_id) continue + if dev_due_count > 0: # perform actual snapshot and create tag: retention + now() as a Y-M-D delete_on_dt = now + retention delete_on = delete_on_dt.strftime('%Y-%m-%d') - volume_data = utils.get_volume(volume_id, region=region) expected_tags = utils.calculate_relevant_tags( - instance_data.get('Tags', None), - volume_data.get('Tags', None)) + instance_data.get('Tags', None)) utils.snapshot_and_tag( instance_id, ami_id, - volume_id, delete_on, region, additional_tags=expected_tags) + else: + LOG.debug('NOT Performing snapshot for %s', instance_id) + continue def should_perform_snapshot(frequency, now, volume_id, recent=None): diff --git a/ebs_snapper/utils.py b/ebs_snapper/utils.py index f6f46e1..7e3a3a5 100644 --- a/ebs_snapper/utils.py +++ b/ebs_snapper/utils.py @@ -49,7 +49,7 @@ "Cluster", "Role", "Customer", "Version", "Billing1", "Billing2", "Billing3", "Billing4", "Billing5" ] -SNAP_DESC_TEMPLATE = "Created from {0} by EbsSnapper({3}) for {1} from {2}" +SNAP_DESC_TEMPLATE = "Created from {0} by EbsSnapper({2}) for {1}" ALLOWED_SNAPSHOT_DELETE_FAILURES = ['InvalidSnapshot.InUse', 'InvalidSnapshot.NotFound'] UNSUPPORTED_REGION_EXCEPTIONS = ['AuthFailure', 'OptInRequired'] @@ -370,16 +370,15 @@ def build_snapshot_paginator(params, region): return paginator.paginate(**params) -def snapshot_and_tag(instance_id, ami_id, volume_id, delete_on, region, additional_tags=None): +def snapshot_and_tag(instance_id, ami_id, delete_on, region, additional_tags=None): """Create snapshot and retention tag""" - LOG.warn('Creating snapshot in %s of volume %s, valid until %s', - region, volume_id, delete_on) + LOG.warn('Creating snapshots in %s for volumes attached to instance %s, valid until %s', + region, instance_id, delete_on) snapshot_description = SNAP_DESC_TEMPLATE.format( instance_id, ami_id, - volume_id, ebs_snapper.__version__ ) @@ -397,18 +396,22 @@ def snapshot_and_tag(instance_id, ami_id, volume_id, delete_on, region, addition ec2 = boto3.client('ec2', region_name=region) - snapshot = ec2.create_snapshot( - VolumeId=volume_id, - Description=snapshot_description[0:254] + snapshot = ec2.create_snapshots( + InstanceSpecification={ + 'InstanceId': instance_id, + 'ExcludeBootVolume': False + }, + Description=snapshot_description[0:254], + TagSpecifications=[ + { + 'ResourceType': "snapshot", + 'Tags': full_tags[:50] + }, + ], ) - ec2.create_tags( - Resources=[snapshot['SnapshotId']], - Tags=full_tags[:50] - ) - - LOG.debug('Finished snapshot in %s of volume %s, valid until %s', - region, volume_id, delete_on) + LOG.debug('Finished snapshots in %s attached to instance %s, valid until %s', + region, instance_id, delete_on) def delete_snapshot(snapshot_id, region): @@ -515,7 +518,7 @@ def get_snapshot_settings_by_instance(instance_id, configurations, region): return None -def calculate_relevant_tags(instance_tags, volume_tags, max_results=50): +def calculate_relevant_tags(instance_tags, max_results=50): """Copy AWS tags from instance to volume to snapshot, per product guide""" # ordered dict of tags, because we care about order @@ -532,13 +535,6 @@ def calculate_relevant_tags(instance_tags, volume_tags, max_results=50): tag_name, tag_value = tag_ds['Key'], tag_ds['Value'] calculated_tags[tag_name] = tag_value - # overwrite tag values from instances with volume tags/values - if volume_tags is not None: - # add relevant ones to the list - for tag_ds in volume_tags: - tag_name, tag_value = tag_ds['Key'], tag_ds['Value'] - calculated_tags[tag_name] = tag_value - returned_tags = [] for n, v in calculated_tags.iteritems(): # skip any tags that were None/falsey, and don't go above max_results diff --git a/requirements.txt b/requirements.txt index dc0ef5d..24ab984 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -boto3==1.7.4 +boto3==1.9.183 boto==2.47.0 -botocore==1.10.4 +botocore==1.12.183 crontab==0.21.3 lambda-uploader==1.1.0 pytimeparse==1.1.5