path: root/rdiff-backup/rdiff_backup/Main.py

# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# rdiff-backup is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with rdiff-backup; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA

"""Start (and end) here - read arguments, set global settings, etc."""

from __future__ import generators
import getopt, sys, re, os
from log import Log, LoggerError
import Globals, Time, SetConnections, selection, robust, rpath, \
	   manage, backup, connection, restore, FilenameMapping, \
	   Security, Hardlink, regress, C


action = None
remote_cmd, remote_schema = None, None
force = None
select_opts, select_mirror_opts = [], []
select_files = []

def parse_cmdlineoptions(arglist):
	"""Parse argument list and set global preferences"""
	global args, action, force, restore_timestr, remote_cmd, remote_schema
	global remove_older_than_string
	def sel_fl(filename):
		"""Helper function for including/excluding filelists below"""
		try: return open(filename, "r")
		except IOError: Log.FatalError("Error opening file %s" % filename)

	try: optlist, args = getopt.getopt(arglist, "blr:sv:V",
		 ["backup-mode", "calculate-average", "chars-to-quote=",
		  "check-destination-dir", "current-time=", "exclude=",
		  "exclude-device-files", "exclude-filelist=",
		  "exclude-filelist-stdin", "exclude-globbing-filelist=",
		  "exclude-mirror=", "exclude-other-filesystems",
		  "exclude-regexp=", "exclude-special-files", "force",
		  "include=", "include-filelist=", "include-filelist-stdin",
		  "include-globbing-filelist=", "include-regexp=",
		  "list-changed-since=", "list-increments", "no-compression",
		  "no-compression-regexp=", "no-hard-links", "null-separator",
		  "parsable-output", "print-statistics", "quoting-char=",
		  "remote-cmd=", "remote-schema=", "remove-older-than=",
		  "restore-as-of=", "restrict=", "restrict-read-only=",
		  "restrict-update-only=", "server", "sleep-ratio=",
		  "ssh-no-compression", "terminal-verbosity=", "test-server",
		  "verbosity=", "version", "windows-mode",
		  "windows-time-format"])
	except getopt.error, e:
		commandline_error("Bad commandline options: %s" % str(e))

	for opt, arg in optlist:
		if opt == "-b" or opt == "--backup-mode": action = "backup"
		elif opt == "--calculate-average": action = "calculate-average"
		elif opt == "--check-destination-dir": action = "check-destination-dir"
		elif opt == "--chars-to-quote":
			Globals.set('chars_to_quote', arg)
			Globals.set('quoting_enabled', 1)
		elif opt == "--current-time":
			Globals.set_integer('current_time', arg)
		elif opt == "--exclude": select_opts.append((opt, arg))
		elif opt == "--exclude-device-files": select_opts.append((opt, arg))
		elif opt == "--exclude-filelist":
			select_opts.append((opt, arg))
			select_files.append(sel_fl(arg))
		elif opt == "--exclude-filelist-stdin":
			select_opts.append(("--exclude-filelist", "standard input"))
			select_files.append(sys.stdin)
		elif opt == "--exclude-globbing-filelist":
			select_opts.append((opt, arg))
			select_files.append(sel_fl(arg))
		elif opt == "--exclude-mirror":
			select_mirror_opts.append(("--exclude", arg))
		elif (opt == "--exclude-other-filesystems" or
			  opt == "--exclude-regexp" or
			  opt == "--exclude-special-files"): select_opts.append((opt, arg))
		elif opt == "--force": force = 1
		elif opt == "--include": select_opts.append((opt, arg))
		elif opt == "--include-filelist":
			select_opts.append((opt, arg))
			select_files.append(sel_fl(arg))
		elif opt == "--include-filelist-stdin":
			select_opts.append(("--include-filelist", "standard input"))
			select_files.append(sys.stdin)
		elif opt == "--include-globbing-filelist":
			select_opts.append((opt, arg))
			select_files.append(sel_fl(arg))
		elif opt == "--include-regexp": select_opts.append((opt, arg))
		elif opt == "--list-changed-since":
			restore_timestr, action = arg, "list-changed-since"
		elif opt == "-l" or opt == "--list-increments":
			action = "list-increments"
		elif opt == "--no-compression": Globals.set("compression", None)
		elif opt == "--no-compression-regexp":
			Globals.set("no_compression_regexp_string", arg)
		elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
		elif opt == "--null-separator": Globals.set("null_separator", 1)
		elif opt == "--parsable-output": Globals.set('parsable_output', 1)
		elif opt == "--print-statistics": Globals.set('print_statistics', 1)
		elif opt == "--quoting-char":
			Globals.set('quoting_char', arg)
			Globals.set('quoting_enabled', 1)
		elif opt == "-r" or opt == "--restore-as-of":
			restore_timestr, action = arg, "restore-as-of"
		elif opt == "--remote-cmd": remote_cmd = arg
		elif opt == "--remote-schema": remote_schema = arg
		elif opt == "--remove-older-than":
			remove_older_than_string = arg
			action = "remove-older-than"
		elif opt == "--restrict": Globals.restrict_path = arg
		elif opt == "--restrict-read-only":
			Globals.security_level = "read-only"
			Globals.restrict_path = arg
		elif opt == "--restrict-update-only":
			Globals.security_level = "update-only"
			Globals.restrict_path = arg
		elif opt == "-s" or opt == "--server":
			action = "server"
			Globals.server = 1
		elif opt == "--sleep-ratio":
			Globals.set_float("sleep_ratio", arg, 0, 1, inclusive=0)
		elif opt == "--ssh-no-compression":
			Globals.set('ssh_compression', None)
		elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg)
		elif opt == "--test-server": action = "test-server"
		elif opt == "-V" or opt == "--version":
			print "rdiff-backup " + Globals.version
			sys.exit(0)
		elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg)
		elif opt == "--windows-mode":
			Globals.set('chars_to_quote', "A-Z:")
			Globals.set('quoting_enabled', 1)
			Globals.set('preserve_hardlinks', 0)
		else: Log.FatalError("Unknown option %s" % opt)

def isincfilename(path):
	"""Return true if path is of a (possibly quoted) increment file"""
	rp = rpath.RPath(Globals.local_connection, path)
	if Globals.quoting_enabled:
		if not FilenameMapping.quoting_char:
			FilenameMapping.set_init_quote_vals()
		rp = FilenameMapping.get_quotedrpath(rp, separate_basename = 1)
	result = rp.isincfile()
	return result

def set_action():
	"""Check arguments and try to set action"""
	global action
	l = len(args)
	if not action:
		if l == 0: commandline_error("No arguments given")
		elif l == 1: action = "restore"
		elif l == 2:
			if isincfilename(args[0]): action = "restore"
			else: action = "backup"
		else: commandline_error("Too many arguments given")

	if l == 0 and action != "server":
		commandline_error("No arguments given")
	if l > 0 and action == "server":
		commandline_error("Too many arguments given")
	if l < 2 and (action == "backup" or action == "restore-as-of"):
		commandline_error("Two arguments are required (source, destination).")
	if l == 2 and (action == "list-increments" or
				   action == "remove-older-than" or
				   action == "list-changed-since" or
				   action == "check-destination-dir"):
		commandline_error("Only use one argument, "
						  "the root of the backup directory")
	if l > 2 and action != "calculate-average":
		commandline_error("Too many arguments given")

def commandline_error(message):
	sys.stderr.write("Error: %s\n" % message)
	sys.stderr.write("See the rdiff-backup manual page for instructions\n")
	sys.exit(1)

def misc_setup(rps):
	"""Set default change ownership flag, umask, relay regexps"""
	os.umask(077)
	Time.setcurtime(Globals.current_time)
	FilenameMapping.set_init_quote_vals()
	SetConnections.UpdateGlobal("client_conn", Globals.local_connection)
	Globals.postset_regexp('no_compression_regexp',
						   Globals.no_compression_regexp_string)
	for conn in Globals.connections:
		conn.robust.install_signal_handlers()
		conn.Hardlink.initialize_dictionaries()

def take_action(rps):
	"""Do whatever action says"""
	if action == "server":
		connection.PipeConnection(sys.stdin, sys.stdout).Server()
	elif action == "backup": Backup(rps[0], rps[1])
	elif action == "restore": Restore(*rps)
	elif action == "restore-as-of": RestoreAsOf(rps[0], rps[1])
	elif action == "test-server": SetConnections.TestConnections()
	elif action == "list-changed-since": ListChangedSince(rps[0])
	elif action == "list-increments": ListIncrements(rps[0])
	elif action == "remove-older-than": RemoveOlderThan(rps[0])
	elif action == "calculate-average": CalculateAverage(rps)
	elif action == "check-destination-dir": CheckDest(rps[0])
	else: raise AssertionError("Unknown action " + action)

def cleanup():
	"""Do any last minute cleaning before exiting"""
	Log("Cleaning up", 6)
	Log.close_logfile()
	if not Globals.server: SetConnections.CloseConnections()

def Main(arglist):
	"""Start everything up!"""
	parse_cmdlineoptions(arglist)
	set_action()
	cmdpairs = SetConnections.get_cmd_pairs(args, remote_schema, remote_cmd)
	Security.initialize(action, cmdpairs)
	rps = map(SetConnections.cmdpair2rp, cmdpairs)
	misc_setup(rps)
	take_action(rps)
	cleanup()


def Backup(rpin, rpout):
	"""Backup, possibly incrementally, src_path to dest_path."""
	if Globals.quoting_enabled:
		rpout = FilenameMapping.get_quotedrpath(rpout)
	SetConnections.BackupInitConnections(rpin.conn, rpout.conn)
	backup_set_select(rpin)
	backup_init_dirs(rpin, rpout)
	if prevtime:
		rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout)
		Time.setprevtime(prevtime)
		backup.Mirror_and_increment(rpin, rpout, incdir)
		rpout.conn.Main.backup_remove_curmirror_local()
	else:
		backup.Mirror(rpin, rpout)
		rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout)

def backup_set_select(rpin):
	"""Create Select objects on source connection"""
	rpin.conn.backup.SourceStruct.set_source_select(rpin, select_opts,
													*select_files)

def backup_init_dirs(rpin, rpout):
	"""Make sure rpin and rpout are valid, init data dir and logging"""
	global datadir, incdir, prevtime
	if rpout.lstat() and not rpout.isdir():
		if not force: Log.FatalError("Destination %s exists and is not a "
									 "directory" % rpout.path)
		else:
			Log("Deleting %s" % rpout.path, 3)
			rpout.delete()

	if not rpin.lstat():
		Log.FatalError("Source directory %s does not exist" % rpin.path)
	elif not rpin.isdir():
		Log.FatalError("Source %s is not a directory" % rpin.path)

	datadir = rpout.append_path("rdiff-backup-data")
	SetConnections.UpdateGlobal('rbdir', datadir)
	checkdest_if_necessary(rpout)
	incdir = datadir.append_path("increments")
	prevtime = backup_get_mirrortime()

	if rpout.lstat():
		if rpout.isdir() and not rpout.listdir(): # rpout is empty dir
			rpout.chmod(0700) # just make sure permissions aren't too lax
		elif not datadir.lstat() and not force: Log.FatalError(
"""Destination directory

%s

exists, but does not look like a rdiff-backup directory.  Running
rdiff-backup like this could mess up what is currently in it.  If you
want to update or overwrite it, run rdiff-backup with the --force
option.""" % rpout.path)

	if not rpout.lstat():
		try: rpout.mkdir()
		except os.error:
			Log.FatalError("Unable to create directory %s" % rpout.path)
	if not datadir.lstat(): datadir.mkdir()
	if Log.verbosity > 0:
		Log.open_logfile(datadir.append("backup.log"))
	backup_warn_if_infinite_regress(rpin, rpout)

def backup_warn_if_infinite_regress(rpin, rpout):
	"""Warn user if destination area contained in source area"""
	if rpout.conn is rpin.conn: # it's meaningful to compare paths
		if ((len(rpout.path) > len(rpin.path)+1 and
			 rpout.path[:len(rpin.path)] == rpin.path and
			 rpout.path[len(rpin.path)] == '/') or
			(rpin.path == "." and rpout.path[0] != '/' and
			 rpout.path[:2] != '..')):
			# Just a few heuristics, we don't have to get every case
			if Globals.backup_reader.Globals.select_source.Select(rpout): Log(
"""Warning: The destination directory '%s' may be contained in the
source directory '%s'.  This could cause an infinite regress.  You
may need to use the --exclude option.""" % (rpout.path, rpin.path), 2)

def backup_get_mirrortime():
	"""Return time in seconds of previous mirror, or None if cannot"""
	incbase = Globals.rbdir.append_path("current_mirror")
	mirror_rps = restore.get_inclist(incbase)
	assert len(mirror_rps) <= 1, \
		   "Found %s current_mirror rps, expected <=1" % (len(mirror_rps),)
	if mirror_rps: return mirror_rps[0].getinctime()
	else: return None

def backup_touch_curmirror_local(rpin, rpout):
	"""Make a file like current_mirror.time.data to record time

	When doing an incremental backup, this should happen before any
	other writes, and the file should be removed after all writes.
	That way we can tell whether the previous session aborted if there
	are two current_mirror files.

	When doing the initial full backup, the file can be created after
	everything else is in place.

	"""
	mirrorrp = Globals.rbdir.append("current_mirror.%s.%s" % (Time.curtimestr,
															  "data"))
	Log("Touching mirror marker %s" % mirrorrp.path, 6)
	mirrorrp.touch()
	mirrorrp.fsync_with_dir()

def backup_remove_curmirror_local():
	"""Remove the older of the current_mirror files.  Use at end of session"""
	assert Globals.rbdir.conn is Globals.local_connection
	curmir_incs = restore.get_inclist(Globals.rbdir.append("current_mirror"))
	assert len(curmir_incs) == 2
	if curmir_incs[0].getinctime() < curmir_incs[1].getinctime():
		older_inc = curmir_incs[0]
	else: older_inc = curmir_incs[1]

	C.sync() # Make sure everything is written before curmirror is removed
	older_inc.sync_delete()


def Restore(src_rp, dest_rp = None):
	"""Main restoring function

	Here src_rp should be an increment file, and if dest_rp is
	missing it defaults to the base of the increment.

	"""
	rpin, rpout = restore_check_paths(src_rp, dest_rp)
	restore_common(rpin, rpout, rpin.getinctime())

def RestoreAsOf(rpin, target):
	"""Secondary syntax for restore operation

	rpin - RPath of mirror file to restore (not nec. with correct index)
	target - RPath of place to put restored file

	"""
	rpin, rpout = restore_check_paths(rpin, target, 1)
	try: time = Time.genstrtotime(restore_timestr)
	except Time.TimeException, exc: Log.FatalError(str(exc))
	restore_common(rpin, target, time)

def restore_common(rpin, target, time):
	"""Restore operation common to Restore and RestoreAsOf"""
	if target.conn.os.getuid() == 0:
		SetConnections.UpdateGlobal('change_ownership', 1)
	mirror_root, index = restore_get_root(rpin)
	restore_check_backup_dir(mirror_root)
	mirror = mirror_root.new_index(index)
	inc_rpath = datadir.append_path('increments', index)
	restore_init_select(mirror_root, target)
	restore_start_log(rpin, target, time)
	restore.Restore(mirror, inc_rpath, target, time)
	Log("Restore ended", 4)

def restore_start_log(rpin, target, time):
	"""Open restore log file, log initial message"""
	try: Log.open_logfile(datadir.append("restore.log"))
	except LoggerError, e: Log("Warning, " + str(e), 2)

	# Log following message at file verbosity 3, but term verbosity 4
	log_message = ("Starting restore of %s to %s as it was as of %s." %
				   (rpin.path, target.path, Time.timetopretty(time)))
	if Log.term_verbosity >= 4: Log.log_to_term(log_message, 4)
	if Log.verbosity >= 3: Log.log_to_file(log_message)

def restore_check_paths(rpin, rpout, restoreasof = None):
	"""Check paths and return pair of corresponding rps"""
	if not restoreasof:
		if not rpin.lstat():
			Log.FatalError("Source file %s does not exist" % rpin.path)
		if Globals.quoting_enabled:
			rpin = FilenameMapping.get_quotedrpath(rpin, 1)
		if not rpin.isincfile():
			Log.FatalError("""File %s does not look like an increment file.

Try restoring from an increment file (the filenames look like
"foobar.2001-09-01T04:49:04-07:00.diff").""" % rpin.path)

	if not rpout: rpout = rpath.RPath(Globals.local_connection,
									  rpin.getincbase_str())
	if rpout.lstat() and not force:
		Log.FatalError("Restore target %s already exists, "
					   "specify --force to overwrite." % rpout.path)
	return rpin, rpout

def restore_check_backup_dir(rpin):
	"""Make sure backup dir root rpin is in consistent state"""
	result = checkdest_need_check(rpin)
	if result is None:
		Log.FatalError("%s does not appear to be an rdiff-backup directory."
					   % (rpin.path,))
	elif result == 1: Log.FatalError(
		"Previous backup to %s seems to have failed."
		"Rerun rdiff-backup with --check-destination-dir option to revert"
		"directory to state before unsuccessful session." % (rpin.path,))

def restore_init_select(rpin, rpout):
	"""Initialize Select

	Unlike the backup selections, here they are on the local
	connection, because the backup operation is pipelined in a way
	the restore operation isn't.

	"""
	restore._select_mirror = selection.Select(rpin)
	restore._select_mirror.ParseArgs(select_mirror_opts, [])
	restore._select_mirror.parse_rbdir_exclude()
	restore._select_source = selection.Select(rpout)

def restore_get_root(rpin):
	"""Return (mirror root, index) and set the data dir

	The idea here is to keep backing up on the path until we find
	a directory that contains "rdiff-backup-data".  That is the
	mirror root.  If the path from there starts
	"rdiff-backup-data/increments*", then the index is the
	remainder minus that.  Otherwise the index is just the path
	minus the root.

	All this could fail if the increment file is pointed to in a
	funny way, using symlinks or somesuch.

	"""
	global datadir
	if rpin.isincfile(): relpath = rpin.getincbase().path
	else: relpath = rpin.path
	pathcomps = os.path.join(rpin.conn.os.getcwd(), relpath).split("/")
	assert len(pathcomps) >= 2 # path should be relative to /

	i = len(pathcomps)
	while i >= 2:
		parent_dir = rpath.RPath(rpin.conn, "/".join(pathcomps[:i]))
		if (parent_dir.isdir() and
			"rdiff-backup-data" in parent_dir.listdir()): break
		i = i-1
	else: Log.FatalError("Unable to find rdiff-backup-data directory")

	if not Globals.quoting_enabled: rootrp = parent_dir
	else: rootrp = FilenameMapping.get_quotedrpath(parent_dir)
	Log("Using mirror root directory %s" % rootrp.path, 6)

	datadir = rootrp.append_path("rdiff-backup-data")
	SetConnections.UpdateGlobal('rbdir', datadir)
	if not datadir.isdir():
		Log.FatalError("Unable to read rdiff-backup-data directory %s" %
					   datadir.path)

	from_datadir = tuple(pathcomps[i:])
	if not from_datadir or from_datadir[0] != "rdiff-backup-data":
		return (rootrp, from_datadir) # in mirror, not increments
	assert from_datadir[1] == "increments"
	return (rootrp, from_datadir[2:])


def ListIncrements(rp):
	"""Print out a summary of the increments and their times"""
	mirror_root, index = restore_get_root(rp)
	restore_check_backup_dir(mirror_root)
	mirror_rp = mirror_root.new_index(index)
	inc_rpath = Globals.rbdir.append_path('increments', index)
	incs = restore.get_inclist(inc_rpath)
	mirror_time = restore.MirrorStruct.get_mirror_time()
	if Globals.parsable_output:
		print manage.describe_incs_parsable(incs, mirror_time, mirror_rp)
	else: print manage.describe_incs_human(incs, mirror_time, mirror_rp)


def CalculateAverage(rps):
	"""Print out the average of the given statistics files"""
	statobjs = map(lambda rp: StatsObj().read_stats_from_rp(rp), rps)
	average_stats = StatsObj().set_to_average(statobjs)
	print average_stats.get_stats_logstring(
		"Average of %d stat files" % len(rps))


def RemoveOlderThan(rootrp):
	"""Remove all increment files older than a certain time"""
	rom_check_dir(rootrp)
	try: time = Time.genstrtotime(remove_older_than_string)
	except Time.TimeException, exc: Log.FatalError(str(exc))
	timep = Time.timetopretty(time)
	Log("Deleting increment(s) before %s" % timep, 4)

	times_in_secs = [inc.getinctime() for inc in 
					 restore.get_inclist(datadir.append_path("increments"))]
	times_in_secs = filter(lambda t: t < time, times_in_secs)
	if not times_in_secs:
		Log.FatalError("No increments older than %s found" % timep)

	times_in_secs.sort()
	inc_pretty_time = "\n".join(map(Time.timetopretty, times_in_secs))
	if len(times_in_secs) > 1 and not force:
		Log.FatalError("Found %d relevant increments, dated:\n%s"
			"\nIf you want to delete multiple increments in this way, "
			"use the --force." % (len(times_in_secs), inc_pretty_time))

	if len(times_in_secs) == 1:
		Log("Deleting increment at time:\n" + inc_pretty_time, 3)
	else: Log("Deleting increments at times:\n" + inc_pretty_time, 3)
	manage.delete_earlier_than(datadir, time)

def rom_check_dir(rootrp):
	"""Check destination dir before RemoveOlderThan"""
	SetConnections.UpdateGlobal('rbdir',
								rootrp.append_path("rdiff-backup-data"))
	if not Globals.rbdir.isdir():
		Log.FatalError("Unable to open rdiff-backup-data dir %s" %
					   (datadir.path,))
	checkdest_if_necessary(rootrp)


def ListChangedSince(rp):
	"""List all the files under rp that have changed since restoretime"""
	try: rest_time = Time.genstrtotime(restore_timestr)
	except Time.TimeException, exc: Log.FatalError(str(exc))
	mirror_root, index = restore_get_root(rp)
	restore_check_backup_dir(mirror_root)
	mirror_rp = mirror_root.new_index(index)
	inc_rp = mirror_rp.append_path("increments", index)
	restore.ListChangedSince(mirror_rp, inc_rp, rest_time)


def CheckDest(dest_rp):
	"""Check the destination directory, """
	need_check = checkdest_need_check(dest_rp)
	if need_check is None:
		Log.FatalError("No destination dir found at %s" % (dest_rp.path,))
	elif need_check == 0:
		Log.FatalError("Destination dir %s does not need checking" %
					   (dest_rp.path,))
	regress.Regress(dest_rp)

def checkdest_need_check(dest_rp):
	"""Return None if no dest dir found, 1 if dest dir needs check, 0 o/w"""
	assert dest_rp.conn is Globals.rbdir.conn
	if not dest_rp.isdir() or not Globals.rbdir.isdir(): return None
	curmirroot = Globals.rbdir.append("current_mirror")
	curmir_incs = restore.get_inclist(curmirroot)
	if not curmir_incs: return None
	elif len(curmir_incs) == 1: return 0
	else:
		assert len(curmir_incs) == 2, "Found too many current_mirror incs!"
		return 1

def checkdest_if_necessary(dest_rp):
	"""Check the destination dir if necessary.

	This can/should be run before an incremental backup.

	"""
	need_check = checkdest_need_check(dest_rp)
	if need_check == 1:
		Log("Previous backup seems to have failed, checking now.", 2)
		regress.Regress(dest_rp)