Skip to content
This repository was archived by the owner on Jul 28, 2021. It is now read-only.

Update to use per instance , multi volume, crash-consistent snapshots #98

Open
smllycat wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Update to use per instance , multi volume, crash-consistent snapshots #98

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions ebs_snapper/snapshot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,11 @@ def perform_snapshot(context, region, installed_region='us-east-1'):
ami_id = instance_data['ImageId']
LOG.info('Reviewing snapshots in region %s on instance %s', region, instance_id)

# grabbing nuber of volumes attached. If there is a volume that is atttached
# which has at least one snapshot due, it will call the function
dev = instance_data.get('BlockDeviceMappings', [])
dev_due_count = 0

for dev in instance_data.get('BlockDeviceMappings', []):
# before we go make a bunch more API calls
if timeout_check(context, 'perform_snapshot'):
Expand All @@ -136,36 +141,35 @@ def perform_snapshot(context, region, installed_region='us-east-1'):
LOG.debug('Considering device %s', dev)
volume_id = dev['Ebs']['VolumeId']

if volume_id in ignore_ids:
continue

# find snapshots
recent = volume_snap_recent.get(volume_id)
now = datetime.datetime.now(dateutil.tz.tzutc())

# snapshot due?
if should_perform_snapshot(frequency, now, volume_id, recent):
LOG.debug('Performing snapshot for %s, calculating tags', volume_id)
dev_due_count = dev_due_count +1
else:
LOG.debug('NOT Performing snapshot for %s', volume_id)
LOG.debug('Snapshot for %s is not due at this time', volume_id)
continue

if dev_due_count > 0:
# perform actual snapshot and create tag: retention + now() as a Y-M-D
delete_on_dt = now + retention
delete_on = delete_on_dt.strftime('%Y-%m-%d')

volume_data = utils.get_volume(volume_id, region=region)
expected_tags = utils.calculate_relevant_tags(
instance_data.get('Tags', None),
volume_data.get('Tags', None))
instance_data.get('Tags', None))

utils.snapshot_and_tag(
instance_id,
ami_id,
volume_id,
delete_on,
region,
additional_tags=expected_tags)
else:
LOG.debug('NOT Performing snapshot for %s', instance_id)
continue


def should_perform_snapshot(frequency, now, volume_id, recent=None):
Expand Down
42 changes: 19 additions & 23 deletions ebs_snapper/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
"Cluster", "Role", "Customer", "Version",
"Billing1", "Billing2", "Billing3", "Billing4", "Billing5"
]
SNAP_DESC_TEMPLATE = "Created from {0} by EbsSnapper({3}) for {1} from {2}"
SNAP_DESC_TEMPLATE = "Created from {0} by EbsSnapper({2}) for {1}"
ALLOWED_SNAPSHOT_DELETE_FAILURES = ['InvalidSnapshot.InUse', 'InvalidSnapshot.NotFound']
UNSUPPORTED_REGION_EXCEPTIONS = ['AuthFailure', 'OptInRequired']

Expand Down Expand Up @@ -370,16 +370,15 @@ def build_snapshot_paginator(params, region):
return paginator.paginate(**params)


def snapshot_and_tag(instance_id, ami_id, volume_id, delete_on, region, additional_tags=None):
def snapshot_and_tag(instance_id, ami_id, delete_on, region, additional_tags=None):
"""Create snapshot and retention tag"""

LOG.warn('Creating snapshot in %s of volume %s, valid until %s',
region, volume_id, delete_on)
LOG.warn('Creating snapshots in %s for volumes attached to instance %s, valid until %s',
region, instance_id, delete_on)

snapshot_description = SNAP_DESC_TEMPLATE.format(
instance_id,
ami_id,
volume_id,
ebs_snapper.__version__
)

Expand All @@ -397,18 +396,22 @@ def snapshot_and_tag(instance_id, ami_id, volume_id, delete_on, region, addition

ec2 = boto3.client('ec2', region_name=region)

snapshot = ec2.create_snapshot(
VolumeId=volume_id,
Description=snapshot_description[0:254]
snapshot = ec2.create_snapshots(
InstanceSpecification={
'InstanceId': instance_id,
'ExcludeBootVolume': False
},
Description=snapshot_description[0:254],
TagSpecifications=[
{
'ResourceType': "snapshot",
'Tags': full_tags[:50]
},
],
)

ec2.create_tags(
Resources=[snapshot['SnapshotId']],
Tags=full_tags[:50]
)

LOG.debug('Finished snapshot in %s of volume %s, valid until %s',
region, volume_id, delete_on)
LOG.debug('Finished snapshots in %s attached to instance %s, valid until %s',
region, instance_id, delete_on)


def delete_snapshot(snapshot_id, region):
Expand Down Expand Up @@ -515,7 +518,7 @@ def get_snapshot_settings_by_instance(instance_id, configurations, region):
return None


def calculate_relevant_tags(instance_tags, volume_tags, max_results=50):
def calculate_relevant_tags(instance_tags, max_results=50):
"""Copy AWS tags from instance to volume to snapshot, per product guide"""

# ordered dict of tags, because we care about order
Expand All @@ -532,13 +535,6 @@ def calculate_relevant_tags(instance_tags, volume_tags, max_results=50):
tag_name, tag_value = tag_ds['Key'], tag_ds['Value']
calculated_tags[tag_name] = tag_value

# overwrite tag values from instances with volume tags/values
if volume_tags is not None:
# add relevant ones to the list
for tag_ds in volume_tags:
tag_name, tag_value = tag_ds['Key'], tag_ds['Value']
calculated_tags[tag_name] = tag_value

returned_tags = []
for n, v in calculated_tags.iteritems():
# skip any tags that were None/falsey, and don't go above max_results
Expand Down
4 changes: 2 additions & 2 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
boto3==1.7.4
boto3==1.9.183
boto==2.47.0
botocore==1.10.4
botocore==1.12.183
crontab==0.21.3
lambda-uploader==1.1.0
pytimeparse==1.1.5