tools

reflink-snap.rb (9289B)

---

 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
 #
 # reflink-snap.rb — reflink (CoW) snapshot manager for XFS.
 #
 # Ruby port of alexadriaanse/reflink-snap (create/prune/latest logic), plus
 # extent-sharing verification and dedup from
 # https://strugglers.net/posts/2017/xfs-reflinks-and-deduplication/
 #
 # A snapshot is a plain directory <snapshot_dir>/<prefix><timestamp> copied
 # with `cp -a --reflink=always -x`, so unmodified data shares extents with
 # the source and costs (almost) no disk space. `latest` symlinks the newest.
 
 require "find"
 require "open3"
 require "optparse"
 
 TS_FORMAT = "%Y-%m-%d_%H%M%S"
 TS_RE = /\d{4}-\d{2}-\d{2}_\d{6}/
 LATEST = "latest"
 
 BANNER = <<~EOS
   reflink-snap.rb — reflink (CoW) snapshots for XFS
 
   Usage:
     reflink-snap.rb create -s SRC -d SNAPDIR [-c KEEP] [-p PREFIX] [-n]
     reflink-snap.rb list   -d SNAPDIR [-p PREFIX]
     reflink-snap.rb prune  -d SNAPDIR -c KEEP [-p PREFIX] [-n]
     reflink-snap.rb verify PATH
     reflink-snap.rb dedupe [--hashfile FILE] PATH...
 
   Commands:
     create   copy SRC to SNAPDIR/[PREFIX]<timestamp> via cp -a --reflink=always,
              update the `latest` symlink, then prune if -c is given
     list     show snapshots (optionally filtered by -p), newest last
     prune    keep the newest KEEP snapshots, delete the rest
     verify   report how much of PATH's file data lives in shared (reflinked)
              extents, using filefrag -v
     dedupe   run duperemove to turn duplicate blocks under PATH... into reflinks
 
   Restoring is just a copy back out of a snapshot:
     cp -a --reflink=always SNAPDIR/latest/some/file SRC/some/file
 
   Space accounting: `du` counts reflinked data once per tree, so snapshots
   look expensive; `df` is the truth for what they actually cost.
 
   Example fcron entries:
     0 * * * *  ruby reflink-snap.rb create -s /home/miro -d /home/.snapshots -c 48 -p hourly_
     0 3 * * *  ruby reflink-snap.rb create -s /home/miro -d /home/.snapshots -c 14 -p daily_
 
   Options:
 EOS
 
 def die(msg)
   abort("reflink-snap: #{msg}")
 end
 
 def human(bytes)
   units = %w[B KiB MiB GiB TiB]
   size = bytes.to_f
   unit = units.first
   units.each do |u|
     unit = u
     break if size < 1024
     size /= 1024
   end
   format("%.1f %s", size, unit)
 end
 
 class ReflinkSnap
   def initialize(opts)
     @opts = opts
     @dry_run = opts.fetch(:dry_run, false)
   end
 
   def create
     src, dir, prefix = @opts[:src], @opts[:dir], @opts[:prefix]
     die "source #{src} is not a directory" unless File.directory?(src)
 
     run "mkdir", "-p", "--", dir
     if Dir.exist?(dir) && File.stat(dir).dev != File.stat(src).dev
       die "#{dir} is on a different filesystem than #{src}; reflinks cannot cross filesystems"
     end
 
     src_real = File.realpath(src)
     snap_real = Dir.exist?(dir) ? File.realpath(dir) : File.expand_path(dir)
     if snap_real.start_with?("#{src_real}/") && File.dirname(snap_real) != src_real
       warn "reflink-snap: warning: #{dir} is nested below #{src} but is not a" \
         " direct child, so snapshots will recursively include older snapshots"
     end
 
     src_dev = File.stat(src).dev
     entries = Dir.children(src).sort.filter_map do |name|
       path = File.join(src, name)
       next if File.lstat(path).dev != src_dev # skip mountpoints, like cp -x
       real = begin
         File.realpath(path)
       rescue Errno::ENOENT
         nil # broken symlink; cp -a copies the link itself, keep it
       end
       next if real == snap_real # never snapshot the snapshot dir
       path
     end
     die "nothing to snapshot in #{src}" if entries.empty?
 
     name = "#{prefix}#{Time.now.strftime(TS_FORMAT)}"
     final = File.join(dir, name)
     die "#{final} already exists (snapshots have 1s resolution)" if File.exist?(final)
 
     # Copy into a dot-prefixed temp dir and rename at the end, so an
     # interrupted copy never looks like a valid snapshot.
     tmp = File.join(dir, ".incomplete_#{name}")
     puts "Creating snapshot #{final}"
     run "mkdir", "--", tmp
     begin
       run "cp", "-a", "--reflink=always", "-x", "--", *entries, "#{tmp}/"
     rescue => e
       system("rm", "-rf", "--", tmp) unless @dry_run
       die "cp failed (#{e.message}); incomplete snapshot removed"
     end
     run "mv", "-T", "--", tmp, final
     run "ln", "-sfn", "--", name, File.join(dir, LATEST)
     puts "Done"
 
     prune if @opts[:count]
   end
 
   def list
     dir, prefix = @opts[:dir], @opts[:prefix]
     snaps = snapshots(dir, prefix)
     if snaps.empty?
       puts prefix ? "No #{prefix}* snapshots in #{dir}" : "No snapshots in #{dir}"
       return
     end
     latest = begin
       File.readlink(File.join(dir, LATEST))
     rescue
       nil
     end
     snaps.each do |name|
       mtime = File.stat(File.join(dir, name)).mtime.strftime("%Y-%m-%d %H:%M:%S")
       puts "#{name}  #{mtime}#{"  (latest)" if name == latest}"
     end
   end
 
   def prune
     dir, prefix, keep = @opts[:dir], @opts[:prefix], @opts[:count]
     die "prune needs -c KEEP" unless keep
     die "-c KEEP must be at least 1" if keep < 1
 
     snaps = snapshots(dir, prefix)
     doomed = (snaps.size > keep) ? snaps.first(snaps.size - keep) : []
     doomed.each do |name|
       run "rm", "-rf", "--", File.join(dir, name)
       puts "Removed snapshot #{File.join(dir, name)}" unless @dry_run
     end
 
     latest = File.join(dir, LATEST)
     if !@dry_run && File.symlink?(latest) && !File.exist?(latest)
       newest = snapshots(dir, prefix).last
       newest ? run("ln", "-sfn", "--", newest, latest) : run("rm", "--", latest)
     end
   end
 
   def verify
     path = @opts[:path]
     die "#{path} does not exist" unless File.exist?(path)
     path = File.realpath(path) # so verifying the `latest` symlink works
     unless system("filefrag", "-V", out: File::NULL, err: File::NULL)
       die "filefrag not found; install e2fsprogs"
     end
 
     files = []
     Find.find(path) do |p|
       next if File.symlink?(p)
       files << p if File.file?(p) && File.size(p).positive?
     end
     die "no regular files under #{path}" if files.empty?
 
     total_bytes = shared_bytes = 0
     files.each_slice(200) do |batch|
       # -b1 reports extent lengths in bytes, so no block-size math is needed.
       out, = Open3.capture3("filefrag", "-v", "-b1", *batch)
       out.each_line do |line|
         next unless line.start_with?(/\s*\d+:/)
         fields = line.split(":")
         bytes = fields[3].to_i
         total_bytes += bytes
         shared_bytes += bytes if fields.last.split(",").include?("shared")
       end
     end
 
     pct = total_bytes.zero? ? 0.0 : 100.0 * shared_bytes / total_bytes
     puts "#{files.size} files, #{human(total_bytes)} of file data"
     puts "  shared (reflinked) extents: #{human(shared_bytes)} (#{pct.round(1)}%)"
     puts "  unique to this tree:        #{human(total_bytes - shared_bytes)}"
   end
 
   def dedupe
     unless system("duperemove", "--help", out: File::NULL, err: File::NULL)
       die "duperemove not installed (xbps-install duperemove)"
     end
     argv = %w[duperemove -hdr]
     argv << "--hashfile=#{@opts[:hashfile]}" if @opts[:hashfile]
     argv += ["--", *@opts[:paths]]
     if @dry_run
       puts "+ #{argv.join(" ")}"
     else
       exec(*argv)
     end
   end
 
   private
 
   # Print instead of executing under -n; raise if a real command fails.
   def run(*argv)
     if @dry_run
       puts "+ #{argv.join(" ")}"
     else
       system(*argv, exception: true)
     end
   end
 
   def name_re(prefix)
     /\A#{Regexp.escape(prefix)}#{TS_RE}\z/
   end
 
   # Directories in `dir` that we created (prefix + timestamp), oldest first.
   # Only these are ever eligible for deletion. When prefix is nil, match any
   # name ending in a timestamp.
   def snapshots(dir, prefix)
     return [] unless Dir.exist?(dir)
     re = prefix ? name_re(prefix) : /\A.*#{TS_RE}\z/o
     Dir.children(dir)
       .grep(re)
       .select { |n| File.directory?(File.join(dir, n)) }
       .sort
   end
 end
 
 opts = {}
 parser = OptionParser.new do |o|
   o.banner = BANNER
   o.on("-s", "--source DIR", "tree to snapshot (create)") { opts[:src] = it }
   o.on("-d", "--snapshot-dir DIR", "where snapshots live") { opts[:dir] = it }
   o.on("-c", "--keep N", Integer, "snapshots to retain") { opts[:count] = it }
   o.on("-p", "--prefix PREFIX", "snapshot name prefix") { opts[:prefix] = it }
   o.on("--hashfile FILE", "duperemove hash cache (dedupe)") { opts[:hashfile] = it }
   o.on("-n", "--dry-run", "print commands instead of running them") { opts[:dry_run] = true }
   o.on("-h", "--help") do
     puts parser
     exit
   end
 end
 
 command = ARGV.shift
 begin
   parser.parse!(ARGV)
 rescue OptionParser::ParseError => e
   die "#{e.message}\n\n#{parser}"
 end
 die "prefix must not contain '/'" if opts[:prefix]&.include?("/")
 
 tool = ReflinkSnap.new(opts)
 
 case command
 when "create"
   die "create needs -s SRC and -d SNAPDIR" unless opts[:src] && opts[:dir]
   tool.create
 when "list"
   die "list needs -d SNAPDIR" unless opts[:dir]
   tool.list
 when "prune"
   die "prune needs -d SNAPDIR" unless opts[:dir]
   tool.prune
 when "verify"
   opts[:path] = ARGV.shift or die "verify needs a PATH"
   tool.verify
 when "dedupe"
   opts[:paths] = ARGV
   die "dedupe needs at least one PATH" if opts[:paths].empty?
   tool.dedupe
 when nil
   puts parser
   exit 1
 else
   die "unknown command #{command}\n\n#{parser}"
 end