diff options
author | bescoto <bescoto@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109> | 2003-06-30 21:24:14 +0000 |
---|---|---|
committer | bescoto <bescoto@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109> | 2003-06-30 21:24:14 +0000 |
commit | a6220370a12ca72b6ddad9627fa77e19d16a7b1a (patch) | |
tree | 8241c228f9c8637ee959675c7007e6bc500842e3 | |
parent | 4488947f39adcd35cf2b4bb167c87aad0d8a92b6 (diff) | |
download | rdiff-backup-a6220370a12ca72b6ddad9627fa77e19d16a7b1a.tar.gz |
Added --list-increment-sizes option
git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@336 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
-rw-r--r-- | rdiff-backup/CHANGELOG | 18 | ||||
-rw-r--r-- | rdiff-backup/rdiff-backup.1 | 10 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/Main.py | 56 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/Security.py | 8 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/manage.py | 76 | ||||
-rw-r--r-- | rdiff-backup/testing/finaltest.py | 56 |
6 files changed, 195 insertions, 29 deletions
diff --git a/rdiff-backup/CHANGELOG b/rdiff-backup/CHANGELOG index 804c873..0a39f78 100644 --- a/rdiff-backup/CHANGELOG +++ b/rdiff-backup/CHANGELOG @@ -1,3 +1,21 @@ +New in v0.13.0 (2003/07/02) +--------------------------- + +To prevent the buildup of confusing and error-prone options, the +capabilities of the source and destination file systems are now +autodetected. Detected features include allowed characters, extended +attributes, access control lists, hard links, ownership, and directory +fsyncing. Options such as --windows-mode, --chars-to-quote, +--quoting-char, and --windows-restore-mode have been removed. + +Now rdiff-backup supports extended attributes. To take advantage of +this you will need the python module pyxattr and a file system that +supports EAs. Thanks to Greg Freemyer for discussion. + +Added --list-increment-sizes switch, which tells you how much space +the various backup files take up. (Suggested by Andrew Bressen) + + New in v0.12.0 (2003/06/26) --------------------------- diff --git a/rdiff-backup/rdiff-backup.1 b/rdiff-backup/rdiff-backup.1 index 08469bf..c079032 100644 --- a/rdiff-backup/rdiff-backup.1 +++ b/rdiff-backup/rdiff-backup.1 @@ -11,7 +11,8 @@ rdiff-backup \- local/remote mirror and incremental backup .B {{ -l | --list-increments } .BI "| --remove-older-than " time_interval .BI "| --list-at-time " time -.BI "| --list-changed-since " time } +.BI "| --list-changed-since " time +.B "| --list-increment-sizes "} .BI [[[ user@ ] host2.foo ]:: destination_directory ] .B rdiff-backup --calculate-average @@ -186,6 +187,13 @@ List the number and date of partial incremental backups contained in the specified destination directory. No backup or restore will take place if this option is given. .TP +.B --list-increment-sizes +List the total size of all the increment and mirror files by time. +This may be helpful in deciding how many increments to keep, and when +to --remove-older-than. Specifying a subdirectory is allowable; then +only the sizes of the mirror and increments pertaining to that +subdirectory will be listed. +.TP .B "-m, --mirror-only" Do not create an rdiff-backup-data directory or make any increments. In this mode rdiff-backup is similar to rsync (but usually diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py index 505025d..f135219 100644 --- a/rdiff-backup/rdiff_backup/Main.py +++ b/rdiff-backup/rdiff_backup/Main.py @@ -55,9 +55,9 @@ def parse_cmdlineoptions(arglist): "include-filelist=", "include-filelist-stdin", "include-globbing-filelist=", "include-regexp=", "list-at-time=", "list-changed-since=", "list-increments", - "no-compare-inode", "no-compression", - "no-compression-regexp=", "no-file-statistics", - "no-hard-links", "null-separator", + "list-increment-sizes", "no-compare-inode", + "no-compression", "no-compression-regexp=", + "no-file-statistics", "no-hard-links", "null-separator", "override-chars-to-quote=", "parsable-output", "print-statistics", "remote-cmd=", "remote-schema=", "remove-older-than=", "restore-as-of=", "restrict=", @@ -105,6 +105,7 @@ def parse_cmdlineoptions(arglist): restore_timestr, action = arg, "list-changed-since" elif opt == "-l" or opt == "--list-increments": action = "list-increments" + elif opt == '--list-increment-sizes': action = 'list-increment-sizes' elif opt == "--no-compare-inode": Globals.set("compare_inode", 0) elif opt == "--no-compression": Globals.set("compression", None) elif opt == "--no-compression-regexp": @@ -147,9 +148,9 @@ def check_action(): """Check to make sure action is compatible with args""" global action arg_action_dict = {0: ['server'], - 1: ['list-increments', 'remove-older-than', - 'list-at-time', 'list-changed-since', - 'check-destination-dir'], + 1: ['list-increments', 'list-increment-sizes', + 'remove-older-than', 'list-at-time', + 'list-changed-since', 'check-destination-dir'], 2: ['backup', 'restore', 'restore-as-of']} l = len(args) if not action: assert l == 2, args # cannot tell backup or restore yet @@ -163,7 +164,7 @@ def final_set_action(rps): global action if action: return assert len(rps) == 2, rps - if restore_get_root(rps[0]): action = "restore" + if restore_set_root(rps[0]): action = "restore" else: action = "backup" def commandline_error(message): @@ -194,6 +195,7 @@ def take_action(rps): elif action == "list-at-time": ListAtTime(rps[0]) elif action == "list-changed-since": ListChangedSince(rps[0]) elif action == "list-increments": ListIncrements(rps[0]) + elif action == 'list-increment-sizes': ListIncrementSizes(rps[0]) elif action == "remove-older-than": RemoveOlderThan(rps[0]) elif action == "calculate-average": CalculateAverage(rps) elif action == "check-destination-dir": CheckDest(rps[0]) @@ -373,7 +375,7 @@ def Restore(src_rp, dest_rp, restore_as_of = None): mirror file), dest_rp should be the target rp to be written. """ - if not restore_root_set: assert restore_get_root(src_rp) + if not restore_root_set: assert restore_set_root(src_rp) restore_check_paths(src_rp, dest_rp, restore_as_of) restore_set_fs_globals(Globals.rbdir) src_rp = restore_init_quoting(src_rp) @@ -449,7 +451,7 @@ def restore_check_paths(rpin, rpout, restoreasof = None): Log.FatalError("Restore target %s already exists, " "specify --force to overwrite." % rpout.path) -def restore_check_backup_dir(mirror_root, src_rp, restore_as_of): +def restore_check_backup_dir(mirror_root, src_rp = None, restore_as_of = 1): """Make sure backup dir root rpin is in consistent state""" if not restore_as_of and not src_rp.isincfile(): Log.FatalError("""File %s does not look like an increment file. @@ -466,7 +468,7 @@ Try restoring from an increment file (the filenames look like "rdiff-with --check-destination-dir option to revert directory " "to state before unsuccessful session." % (mirror_root.path,)) -def restore_get_root(rpin): +def restore_set_root(rpin): """Set data dir, restore_root and index, or return None if fail The idea here is to keep backing up on the path until we find @@ -480,7 +482,7 @@ def restore_get_root(rpin): funny way, using symlinks or somesuch. """ - global restore_root, restore_index + global restore_root, restore_index, restore_root_set if rpin.isincfile(): relpath = rpin.getincbase().path else: relpath = rpin.path pathcomps = os.path.join(rpin.conn.os.getcwd(), relpath).split("/") @@ -510,15 +512,16 @@ def restore_get_root(rpin): (len(from_datadir) == 2 and from_datadir[1].startswith('increments'))), from_datadir restore_index = from_datadir[2:] + restore_root_set = 1 return 1 def ListIncrements(rp): """Print out a summary of the increments and their times""" - mirror_root, index = restore_get_root(rp) - restore_check_backup_dir(mirror_root) - mirror_rp = mirror_root.new_index(index) - inc_rpath = Globals.rbdir.append_path('increments', index) + assert restore_set_root(rp) + restore_check_backup_dir(restore_root) + mirror_rp = restore_root.new_index(restore_index) + inc_rpath = Globals.rbdir.append_path('increments', restore_index) incs = restore.get_inclist(inc_rpath) mirror_time = restore.MirrorStruct.get_mirror_time() if Globals.parsable_output: @@ -526,6 +529,13 @@ def ListIncrements(rp): else: print manage.describe_incs_human(incs, mirror_time, mirror_rp) +def ListIncrementSizes(rp): + """Print out a summary of the increments """ + assert restore_set_root(rp) + restore_check_backup_dir(restore_root) + print manage.ListIncrementSizes(restore_root, restore_index) + + def CalculateAverage(rps): """Print out the average of the given statistics files""" statobjs = map(lambda rp: StatsObj().read_stats_from_rp(rp), rps) @@ -575,10 +585,10 @@ def ListChangedSince(rp): """List all the files under rp that have changed since restoretime""" try: rest_time = Time.genstrtotime(restore_timestr) except Time.TimeException, exc: Log.FatalError(str(exc)) - mirror_root, index = restore_get_root(rp) - restore_check_backup_dir(mirror_root) - mirror_rp = mirror_root.new_index(index) - inc_rp = mirror_rp.append_path("increments", index) + assert restore_set_root(rp) + restore_check_backup_dir(restore_root) + mirror_rp = restore_root.new_index(restore_index) + inc_rp = mirror_rp.append_path("increments", restore_index) restore.ListChangedSince(mirror_rp, inc_rp, rest_time) @@ -586,10 +596,10 @@ def ListAtTime(rp): """List files in archive under rp that are present at restoretime""" try: rest_time = Time.genstrtotime(restore_timestr) except Time.TimeException, exc: Log.FatalError(str(exc)) - mirror_root, index = restore_get_root(rp) - restore_check_backup_dir(mirror_root) - mirror_rp = mirror_root.new_index(index) - inc_rp = mirror_rp.append_path("increments", index) + assert restore_set_root(rp) + restore_check_backup_dir(restore_root) + mirror_rp = restore_root.new_index(restore_index) + inc_rp = mirror_rp.append_path("increments", restore_index) restore.ListAtTime(mirror_rp, inc_rp, rest_time) diff --git a/rdiff-backup/rdiff_backup/Security.py b/rdiff-backup/rdiff_backup/Security.py index 427f826..919261a 100644 --- a/rdiff-backup/rdiff_backup/Security.py +++ b/rdiff-backup/rdiff_backup/Security.py @@ -76,7 +76,7 @@ def set_security_level(action, cmdpairs): rdir = tempfile.gettempdir() elif islocal(cp1): sec_level = "read-only" - Main.restore_get_root(rpath.RPath(Globals.local_connection, + Main.restore_set_root(rpath.RPath(Globals.local_connection, getpath(cp1))) rdir = Main.restore_root.path else: @@ -94,9 +94,9 @@ def set_security_level(action, cmdpairs): assert islocal(cp2) sec_level = "all" rdir = getpath(cp2) - elif (action == "test-server" or action == "list-increments" or - action == "list-at-time" or action == "list-changed-since" - or action == "calculate-average" or action == "remove-older-than"): + elif action in ["test-server", "list-increments", 'list-increment-sizes', + "list-at-time", "list-changed-since", + "calculate-average", "remove-older-than"]: sec_level = "minimal" rdir = tempfile.gettempdir() else: assert 0, "Unknown action %s" % action diff --git a/rdiff-backup/rdiff_backup/manage.py b/rdiff-backup/rdiff_backup/manage.py index bb4259c..bdebae2 100644 --- a/rdiff-backup/rdiff_backup/manage.py +++ b/rdiff-backup/rdiff_backup/manage.py @@ -21,7 +21,7 @@ from __future__ import generators from log import Log -import Globals, Time, static, manage +import Globals, Time, static, manage, statistics, restore, selection class ManageException(Exception): pass @@ -128,3 +128,77 @@ class IncObj: s = ["Increment file %s" % self.incrp.path, "Date: %s" % self.pretty_time()] return "\n".join(s) + + +def ListIncrementSizes(mirror_root, index): + """Return string summarizing the size of all the increments""" + stat_obj = statistics.StatsObj() # used for byte summary string + def get_total(rp_iter): + """Return the total size of everything in rp_iter""" + total = 0 + for rp in rp_iter: total += rp.getsize() + return total + + def get_time_dict(inc_iter): + """Return dictionary pairing times to total size of incs""" + time_dict = {} + for inc in inc_iter: + if not inc.isincfile(): continue + t = inc.getinctime() + if not time_dict.has_key(t): time_dict[t] = 0 + time_dict[t] += inc.getsize() + return time_dict + + def get_mirror_select(): + """Return iterator of mirror rpaths""" + mirror_base = mirror_root.new_index(index) + mirror_select = selection.Select(mirror_base) + if not index: # must exclude rdiff-backup-directory + mirror_select.parse_rbdir_exclude() + return mirror_select.set_iter() + + def get_inc_select(): + """Return iterator of increment rpaths""" + inc_base = Globals.rbdir.append_path('increments', index) + for base_inc in restore.get_inclist(inc_base): yield base_inc + if inc_base.isdir(): + inc_select = selection.Select(inc_base).set_iter() + for inc in inc_select: yield inc + + def get_summary_triples(mirror_total, time_dict): + """Return list of triples (time, size, cumulative size)""" + triples = [] + + cur_mir_base = Globals.rbdir.append('current_mirror') + mirror_time = restore.get_inclist(cur_mir_base)[0].getinctime() + triples.append((mirror_time, mirror_total, mirror_total)) + + inc_times = time_dict.keys() + inc_times.sort() + inc_times.reverse() + cumulative_size = mirror_total + for inc_time in inc_times: + size = time_dict[inc_time] + cumulative_size += size + triples.append((inc_time, size, cumulative_size)) + return triples + + def triple_to_line(triple): + """Convert triple to display string""" + time, size, cum_size = triple + return "%24s %13s %15s" % \ + (Time.timetopretty(time), + stat_obj.get_byte_summary_string(size), + stat_obj.get_byte_summary_string(cum_size)) + + mirror_total = get_total(get_mirror_select()) + time_dict = get_time_dict(get_inc_select()) + triples = get_summary_triples(mirror_total, time_dict) + + l = ['%12s %9s %15s %20s' % ('Time', '', 'Size', 'Cumulative size'), + '-' * 77, + triple_to_line(triples[0]) + ' (current mirror)'] + for triple in triples[1:]: l.append(triple_to_line(triple)) + return '\n'.join(l) + + diff --git a/rdiff-backup/testing/finaltest.py b/rdiff-backup/testing/finaltest.py index 590b80b..aa1239b 100644 --- a/rdiff-backup/testing/finaltest.py +++ b/rdiff-backup/testing/finaltest.py @@ -296,6 +296,62 @@ class Final(PathSetter): assert wc_output.split() == ["0", "0", "0"], wc_output +class FinalMisc(PathSetter): + """Test miscellaneous operations like list-increments, etc. + + Many of these just run and make sure there were no errors; they + don't verify the output. + + """ + def testListIncrementsLocal(self): + """Test --list-increments switch. Assumes restoretest3 valid rd dir""" + self.set_connections(None, None, None, None) + self.exec_rb_extra_args(None, "--list-increments", + "testfiles/restoretest3") + + def testListIncrementsRemote(self): + """Test --list-increments mode remotely. Uses restoretest3""" + self.set_connections('test1', '../', None, None) + self.exec_rb_extra_args(None, "--list-increments", + "testfiles/restoretest3") + + def testListChangeSinceLocal(self): + """Test --list-changed-since mode locally. Uses restoretest3""" + self.set_connections(None, None, None, None) + self.exec_rb_extra_args(None, '--list-changed-since 10000', + 'testfiles/restoretest3') + + def testListChangeSinceRemote(self): + """Test --list-changed-since mode remotely. Uses restoretest3""" + self.set_connections('test1', '../', None, None) + self.exec_rb_extra_args(None, '--list-changed-since 10000', + 'testfiles/restoretest3') + + def testListAtTimeLocal(self): + """Test --list-at-time mode locally. Uses restoretest3""" + self.set_connections(None, None, None, None) + self.exec_rb_extra_args(None, '--list-at-time 20000', + 'testfiles/restoretest3') + + def testListAtTimeRemote(self): + """Test --list-at-time mode locally. Uses restoretest3""" + self.set_connections('test1', '../', None, None) + self.exec_rb_extra_args(None, '--list-at-time 20000', + 'testfiles/restoretest3') + + def testListIncrementSizesLocal(self): + """Test --list-increment-sizes switch. Uses restoretest3""" + self.set_connections(None, None, None, None) + self.exec_rb_extra_args(None, "--list-increment-sizes", + "testfiles/restoretest3") + + def testListIncrementsRemote(self): + """Test --list-increment-sizes mode remotely. Uses restoretest3""" + self.set_connections('test1', '../', None, None) + self.exec_rb_extra_args(None, "--list-increment-sizes", + "testfiles/restoretest3") + + class FinalSelection(PathSetter): """Test selection options""" def run(self, cmd): |