zfs-fetch-pool

#!/usr/bin/env python

from subprocess import Popen,PIPE,check_call,call
import datetime
import sys
import os
import logging
import re
logging.basicConfig()
logging.getLogger().setLevel(logging.WARN)
if "-v" in sys.argv[1:]:
	logging.getLogger().setLevel(logging.INFO)
	sys.argv = [ sys.argv[0] ] + sys.argv[2:]
log = logging.getLogger().info
error = logging.getLogger().error

# to ensure the ZFS utilities are in the path.  Just in case.
os.environ["PATH"] = "/sbin:/usr/sbin:/usr/local/sbin:" + os.environ["PATH"]

try:
	local_pool = sys.argv[1]
	m = re.findall("^(|ssh://|rsh://)([^/.]+)@(.+)/(.+)$",sys.argv[2])[0]
	method,user,host,remote_pool = m
	if not method: method = "ssh"
	elif method == "ssh://": method = "ssh"
	elif method == "rsh://": method = "rsh"
	else: assert False
	log("Connecting through %s as %s to %s and graft %s into %s",
		method,user,host,remote_pool,local_pool)
except (IndexError,ValueError), e:
	error("usage: zfs-fetch-pool [-v] <localpool> <user@host:remotepool>")
	sys.exit(os.EX_USAGE)

local_cmd = ["zfs"]
remote_cmd = [method,"-l",user,host]

def get_todays_snapshot_name():
	d = datetime.date.today()
	name = "%04d-%02d-%02d"%(d.year,d.month,d.day)
	return "zfs-tools-%s"%name

def parse_zfs_list(text):
	datasets = [ line.split()[0] for line in text.splitlines() if line ][1:]
	filesystems = [ f for f in datasets if "@" not in f ]
	snapshots = [ f.split("@",1) for f in datasets if "@" in f ]
	b = {}
	for fs,snapname in snapshots:
		if fs not in b: b[fs] = []
		b[fs].append(snapname)
	return filesystems,b

def get_datasets(cmd):
	return parse_zfs_list(Popen(cmd + ["list"], stdout=PIPE).communicate()[0])

def get_local_datasets(): return get_datasets(local_cmd)
def get_remote_datasets(): return get_datasets(remote_cmd)

def make_snapshot(cmd,pool,name):
	check_call(cmd + ["snapshot","-r","%s@%s"%(pool,name)])

def remove_snapshot(cmd,pool,name):
	check_call(cmd + ["destroy","-r","%s@%s"%(pool,name)])

todays_snapshot = get_todays_snapshot_name()

if not get_remote_datasets()[1] or todays_snapshot not in get_remote_datasets()[1][remote_pool]:
	log("Making today's snapshot in %s"%host)
	make_snapshot(remote_cmd,remote_pool,todays_snapshot)

local_filesystems,local_snapshots = get_local_datasets()
remote_filesystems,remote_snapshots = get_remote_datasets()

for remote_fs in remote_filesystems:
	local_fs = "%s/%s"%(local_pool,remote_fs)

	if local_fs not in local_filesystems:
		# send today's snapshot over there
		log("Receiving %s from %s in full"%(remote_fs,host))
		sendcmd = remote_cmd + [
					"send",
					"%s@%s"%(remote_fs,todays_snapshot)
				]
		receivecmd = local_cmd + [
					"recv",
					"%s@%s"%(local_fs,todays_snapshot)
				]
	else:
		if local_fs not in local_snapshots:
			error("%s exists in local pool %s but has no snapshots"%(remote_fs,local_pool))
			continue

		# if the latest snapshot is the backup server
		if todays_snapshot in local_snapshots[local_fs]:
			# we safely continue to the next item
			log("Skipping %s: today's snapshot already in pool %s"%(local_fs,local_pool))
			continue

		# find latest common ancestor
		last_common_snapshot = None
		for s in [ m for m in local_snapshots[local_fs] if m.startswith("zfs-tools-") ]:
			if s in remote_snapshots[remote_fs]: last_common_snapshot = s
		
		# no common ancestor?
		if not last_common_snapshot:
			# bomb out, motherfucker!
			error("%s exists in local pool %s, but has no common snapshots locally"%remote_fs,local_pool)
			continue

		# and now we're ready to do the incremental backup
		log("Receiving %s from %s incrementally"%(remote_fs,host))
		sendcmd = remote_cmd + [
				"send",
				"-i",
				"@%s"%last_common_snapshot,
				"%s@%s"%(remote_fs,todays_snapshot)
			]
		receivecmd = local_cmd + [
				"recv",
				"-F",
				"%s@%s"%(local_fs,todays_snapshot)
			]

	receive = Popen(receivecmd,stdin=PIPE)
	send = Popen(sendcmd,stdout=receive.stdin)
	receive_ret = receive.wait()
	if receive_ret:
		error("ZFS recv failed with return code %s."%receive_ret)
	send_ret = send.wait()
	if send_ret:
		error("ZFS send failed with return code %s.  Verify if zfs-shell is working."%send_ret)
	if receive_ret != 0 or send_ret != 0:
		ret = [ m for m in [receive_ret,send_ret] if m != 0][0]
		error("Aborting prematurely without destructive action")
		sys.exit(ret)

local_filesystems,local_snapshots = get_local_datasets()
remote_filesystems,remote_snapshots = get_remote_datasets()

for snapshot in [
	    m for m
	    in remote_snapshots["%s"%remote_pool]
	    if m.startswith("zfs-tools-")
	]:
	if snapshot != todays_snapshot:
		log("Removing obsolete remote snapshot %s"%snapshot)
		remove_snapshot(
			remote_cmd,
			remote_pool,
			snapshot)
for snapshot in [
	    m for m
	    in local_snapshots["%s/%s"%(local_pool,remote_pool)]
	    if m.startswith("zfs-tools-")
	]:
	if snapshot != todays_snapshot:
		log("Removing obsolete local snapshot %s"%snapshot)
		remove_snapshot(
			local_cmd,
			"%s/%s"%(local_pool,remote_pool),
			snapshot)