From 9196f61c12238733d43295511b9847cb448da3fb Mon Sep 17 00:00:00 2001 From: Boris Lytochkin Date: Mon, 8 Mar 2021 00:01:41 +0300 Subject: [PATCH] Implement DESTROY_AFTER optional argument for bin/zfs-auto-snapshot. --- README.md | 10 ++++++++-- bin/zfs-auto-snapshot | 12 ++++++++++-- bin/zfs-cleanup-snapshots | 3 +++ lib/zfstools.rb | 29 ++++++++++++++++++++++++++--- lib/zfstools/snapshot.rb | 22 ++++++++++++++++------ 5 files changed, 63 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a037c63..9746f62 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,13 @@ This will handle automatically snapshotting datasets similar to time-sliderd fro ### Usage - /usr/local/bin/zfs-auto-snapshot INTERVAL KEEP + /usr/local/bin/zfs-auto-snapshot INTERVAL KEEP [DESTROY_AFTER] * INTERVAL - The interval for the snapshot. This is something such as `frequent`, `hourly`, `daily`, `weekly`, `monthly`, etc. * KEEP - How many to keep for this INTERVAL. Older ones will be destroyed. +* DESTROY_AFTER - Create snapshot[s] with maximum lifetime of DESTROY_AFTER days starting from invocation timestamp. +Snapshot with an expired `zfstools:destroy_after` property will be deleted upon first `zfs-auto-snapshot` invocation with no relation to KEEP argument. +Userful for snapshots that are created upon non-recurring events (e.g. on boot or manually) so they do not stuck on the pool forever. #### Crontab @@ -36,6 +39,7 @@ This will handle automatically snapshotting datasets similar to time-sliderd fro 7 0 * * * root /usr/local/bin/zfs-auto-snapshot daily 7 14 0 * * 7 root /usr/local/bin/zfs-auto-snapshot weekly 4 28 0 1 * * root /usr/local/bin/zfs-auto-snapshot monthly 12 + @reboot root /usr/local/bin/zfs-auto-snapshot boot 3 30 #### Dataset setup @@ -101,7 +105,9 @@ The `zfs-auto-snapshot` script will automatically flush the tables before saving ### zfs-cleanup-snapshots -Cleans up zero-sized snapshots. This ignores snapshots created by `zfs-auto-snapshot` as it handles zero-sized in its own special way. +Cleans up: +* zero-sized snapshots created not by `zfs-auto-snapshot` as it handles zero-sized in its own special way; +* snapshots with an expired `zfstools:destroy_after` property. #### Usage diff --git a/bin/zfs-auto-snapshot b/bin/zfs-auto-snapshot index ee717c5..26ac4df 100755 --- a/bin/zfs-auto-snapshot +++ b/bin/zfs-auto-snapshot @@ -45,7 +45,7 @@ end def usage puts <<-EOF -Usage: #{$0} [-dknpuv] +Usage: #{$0} [-dknpuv] [DESTROY_AFTER] EOF format = " %-15s %s" puts format % ["-d", "Show debug output."] @@ -58,6 +58,7 @@ Usage: #{$0} [-dknpuv] puts format % ["-v", "Show what is being done."] puts format % ["INTERVAL", "The interval to snapshot."] puts format % ["KEEP", "How many snapshots to keep."] + puts format % ["DESTROY_AFTER", "Keep created stapshots no more than DESTROY_AFTER days. Snapshots will be deleted upon expiration by this utility."] exit end @@ -65,11 +66,18 @@ usage if ARGV.length < 2 interval=ARGV[0] keep=ARGV[1].to_i +destroy_after=nil +if ARGV.length == 3 + destroy_after = (Time.now.to_i + 24*3600*Integer(ARGV[2])) rescue usage +end datasets = find_eligible_datasets(interval, pool) # Generate new snapshots -do_new_snapshots(datasets, interval) if keep > 0 +do_new_snapshots(datasets, interval, destroy_after) if keep > 0 + +# Remove all snapshots with expired destroy_after attribute +cleanup_attr_expired_snapshots(pool) # Delete expired cleanup_expired_snapshots(pool, datasets, interval, keep, should_destroy_zero_sized_snapshots) diff --git a/bin/zfs-cleanup-snapshots b/bin/zfs-cleanup-snapshots index c2afb29..3f1f2eb 100755 --- a/bin/zfs-cleanup-snapshots +++ b/bin/zfs-cleanup-snapshots @@ -53,3 +53,6 @@ snapshots = Zfs::Snapshot.list(pool, {'recursive' => true}).select { |snapshot| datasets = Zfs::Dataset.list(pool) dataset_snapshots = group_snapshots_into_datasets(snapshots, datasets) dataset_snapshots = datasets_destroy_zero_sized_snapshots(dataset_snapshots) + +# Remove all snapshots with expired destroy_after attribute +cleanup_attr_expired_snapshots(pool) diff --git a/lib/zfstools.rb b/lib/zfstools.rb index f31957e..2a8fc2b 100644 --- a/lib/zfstools.rb +++ b/lib/zfstools.rb @@ -28,6 +28,10 @@ def snapshot_format '%Y-%m-%d-%Hh%M' end +def destroy_after_property + "zfstools:destroy_after" +end + ### Get the name of the snapshot to create def snapshot_name(interval) if $use_utc @@ -147,13 +151,16 @@ def find_eligible_datasets(interval, pool) end ### Generate new snapshots -def do_new_snapshots(datasets, interval) +def do_new_snapshots(datasets, interval, destroy_after=nil) snapshot_name = snapshot_name(interval) + options = {} + options['destroy_after'] = destroy_after if destroy_after # Snapshot single - Zfs::Snapshot.create_many(snapshot_name, datasets['single']) + Zfs::Snapshot.create_many(snapshot_name, datasets['single'], options) # Snapshot recursive - Zfs::Snapshot.create_many(snapshot_name, datasets['recursive'], 'recursive'=>true) + options['recursive'] = true + Zfs::Snapshot.create_many(snapshot_name, datasets['recursive'], options) end def group_snapshots_into_datasets(snapshots, datasets) @@ -226,3 +233,19 @@ def cleanup_expired_snapshots(pool, datasets, interval, keep, should_destroy_zer end threads.each { |th| th.join } end + +### Find and destroy snapshots with zfstools:destroy_after attribute expired +### Should be invoked before cleanup_expired_snapshots as we prefer to delete +### snapshots with zfstools:destroy_after expired rather that by count. +def cleanup_attr_expired_snapshots(pool) + current_timestamp = Time.now.to_i; + attr_expired_snapshots = Zfs::Snapshot.list(pool, {'recursive' => true}).select { |snapshot| snapshot.destroy_after?(current_timestamp) } + threads = [] + attr_expired_snapshots.each do |snapshot| + threads << Thread.new do + snapshot.destroy + end + threads.last.join unless $use_threads + end + threads.each { |th| th.join } +end diff --git a/lib/zfstools/snapshot.rb b/lib/zfstools/snapshot.rb index 0551b85..d1b20d7 100644 --- a/lib/zfstools/snapshot.rb +++ b/lib/zfstools/snapshot.rb @@ -6,9 +6,10 @@ module Zfs class Snapshot @@stale_snapshot_size = false attr_reader :name - def initialize(name, used=nil) + def initialize(name, used=nil, destroy_after=nil) @name = name @used = used + @destroy_after = destroy_after end def used @@ -27,19 +28,25 @@ def is_zero? used end + def destroy_after?(timestamp) + return false if @destroy_after.nil? || @destroy_after > timestamp + true + end + ### List all snapshots def self.list(dataset=nil, options={}) snapshots = [] flags=[] flags << "-d 1" if dataset and !options['recursive'] flags << "-r" if options['recursive'] - cmd = "zfs list #{flags.join(" ")} -H -t snapshot -o name,used -S name" + cmd = "zfs list #{flags.join(" ")} -H -t snapshot -o name,used,#{destroy_after_property} -S name" cmd += " " + dataset.shellescape if dataset puts cmd if $debug IO.popen cmd do |io| io.readlines.each do |line| - snapshot_name,used = line.chomp.split("\t") - snapshots << self.new(snapshot_name, used.to_i) + snapshot_name,used,destroy_after = line.chomp.split("\t") + destroy_after = Integer(destroy_after) rescue nil + snapshots << self.new(snapshot_name, used.to_i, destroy_after) end end snapshots @@ -49,6 +56,7 @@ def self.list(dataset=nil, options={}) def self.create(snapshot, options = {}) flags=[] flags << "-r" if options['recursive'] + flags << "-o #{destroy_after_property}=" + options['destroy_after'].to_s if options['destroy_after'] cmd = "zfs snapshot #{flags.join(" ")} " if snapshot.kind_of?(Array) cmd += snapshot.shelljoin @@ -129,9 +137,11 @@ def self.create_many(snapshot_name, datasets, options={}) threads = [] datasets.each do |dataset| threads << Thread.new do - self.create("#{dataset.name}@#{snapshot_name}", + self.create("#{dataset.name}@#{snapshot_name}", { 'recursive' => options['recursive'] || false, - 'db' => dataset.db) + 'db' => dataset.db, + 'destroy_after' => options['destroy_after'] || false, + }) end threads.last.join unless $use_threads end