From 807241bc4f322edc6f95782291900362484263df Mon Sep 17 00:00:00 2001 From: ben Date: Fri, 10 May 2002 23:14:35 +0000 Subject: Lots of changes, see changelog for 0.7.4. git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@72 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/rdiff_backup/connection.py | 45 ++-- rdiff-backup/rdiff_backup/destructive_stepping.py | 63 +++++- rdiff-backup/rdiff_backup/highlevel.py | 47 ++-- rdiff-backup/rdiff_backup/increment.py | 118 ++-------- rdiff-backup/rdiff_backup/lazy.py | 2 +- rdiff-backup/rdiff_backup/manage.py | 79 ++++--- rdiff-backup/rdiff_backup/restore.py | 172 ++++++++++----- rdiff-backup/rdiff_backup/robust.py | 4 +- rdiff-backup/rdiff_backup/rpath.py | 34 ++- rdiff-backup/rdiff_backup/selection.py | 35 +-- rdiff-backup/src/Make | 6 +- rdiff-backup/src/connection.py | 45 ++-- rdiff-backup/src/destructive_stepping.py | 63 +++++- rdiff-backup/src/globals.py | 22 +- rdiff-backup/src/hardlink.py | 38 ++-- rdiff-backup/src/highlevel.py | 47 ++-- rdiff-backup/src/increment.py | 118 ++-------- rdiff-backup/src/lazy.py | 2 +- rdiff-backup/src/main.py | 249 ++++++++++++---------- rdiff-backup/src/manage.py | 79 ++++--- rdiff-backup/src/restore.py | 172 ++++++++++----- rdiff-backup/src/robust.py | 4 +- rdiff-backup/src/rpath.py | 34 ++- rdiff-backup/src/selection.py | 35 +-- rdiff-backup/src/ttime.py | 9 +- 25 files changed, 879 insertions(+), 643 deletions(-) diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py index 9842480..57d2fa5 100644 --- a/rdiff-backup/rdiff_backup/connection.py +++ b/rdiff-backup/rdiff_backup/connection.py @@ -92,6 +92,7 @@ class LowLevelPipeConnection(Connection): b - string q - quit signal t - TempFile + d - DSRPath R - RPath r - RORPath only c - PipeConnection object @@ -118,6 +119,7 @@ class LowLevelPipeConnection(Connection): if type(obj) is types.StringType: self._putbuf(obj, req_num) elif isinstance(obj, Connection): self._putconn(obj, req_num) elif isinstance(obj, TempFile): self._puttempfile(obj, req_num) + elif isinstance(obj, DSRPath): self._putdsrpath(obj, req_num) elif isinstance(obj, RPath): self._putrpath(obj, req_num) elif isinstance(obj, RORPath): self._putrorpath(obj, req_num) elif ((hasattr(obj, "read") or hasattr(obj, "write")) @@ -148,6 +150,11 @@ class LowLevelPipeConnection(Connection): tempfile.index, tempfile.data) self._write("t", cPickle.dumps(tf_repr, 1), req_num) + def _putdsrpath(self, dsrpath, req_num): + """Put DSRPath into pipe. See _putrpath""" + dsrpath_repr = (dsrpath.conn.conn_number, dsrpath.getstatedict()) + self._write("d", cPickle.dumps(dsrpath_repr, 1), req_num) + def _putrpath(self, rpath, req_num): """Put an rpath into the pipe @@ -219,23 +226,22 @@ class LowLevelPipeConnection(Connection): ord(header_string[1]), self._s2l(header_string[2:])) except IndexError: raise ConnectionError() - if format_string == "o": result = cPickle.loads(self._read(length)) - elif format_string == "b": result = self._read(length) - elif format_string == "f": - result = VirtualFile(self, int(self._read(length))) + if format_string == "q": raise ConnectionQuit("Received quit signal") + + data = self._read(length) + if format_string == "o": result = cPickle.loads(data) + elif format_string == "b": result = data + elif format_string == "f": result = VirtualFile(self, int(data)) elif format_string == "i": - result = RORPIter.FromFile(BufferedRead( - VirtualFile(self, int(self._read(length))))) - elif format_string == "t": - result = self._gettempfile(self._read(length)) - elif format_string == "r": - result = self._getrorpath(self._read(length)) - elif format_string == "R": result = self._getrpath(self._read(length)) - elif format_string == "c": - result = Globals.connection_dict[int(self._read(length))] + result = RORPIter.FromFile(BufferedRead(VirtualFile(self, + int(data)))) + elif format_string == "t": result = self._gettempfile(data) + elif format_string == "r": result = self._getrorpath(data) + elif format_string == "R": result = self._getrpath(data) + elif format_string == "d": result = self._getdsrpath(data) else: - assert format_string == "q", header_string - raise ConnectionQuit("Received quit signal") + assert format_string == "c", header_string + result = Globals.connection_dict[int(data)] Log.conn("received", result, req_num) return (req_num, result) @@ -255,6 +261,15 @@ class LowLevelPipeConnection(Connection): conn_number, base, index, data = cPickle.loads(raw_rpath_buf) return RPath(Globals.connection_dict[conn_number], base, index, data) + def _getdsrpath(self, raw_dsrpath_buf): + """Return DSRPath object indicated by buf""" + conn_number, state_dict = cPickle.loads(raw_dsrpath_buf) + empty_dsrp = DSRPath("bypass", Globals.local_connection, None) + empty_dsrp.__setstate__(state_dict) + empty_dsrp.conn = Globals.connection_dict[conn_number] + empty_dsrp.file = None + return empty_dsrp + def _close(self): """Close the pipes associated with the connection""" self.outpipe.close() diff --git a/rdiff-backup/rdiff_backup/destructive_stepping.py b/rdiff-backup/rdiff_backup/destructive_stepping.py index c5e2faa..ff3b42a 100644 --- a/rdiff-backup/rdiff_backup/destructive_stepping.py +++ b/rdiff-backup/rdiff_backup/destructive_stepping.py @@ -1,4 +1,5 @@ from __future__ import generators +import types execfile("rorpiter.py") ####################################################################### @@ -40,13 +41,17 @@ class DSRPath(RPath): otherwise use the same arguments as the RPath initializer. """ - if len(args) == 2 and isinstance(args[0], RPath): + if len(args) == 1 and isinstance(args[0], RPath): rp = args[0] RPath.__init__(self, rp.conn, rp.base, rp.index) else: RPath.__init__(self, *args) - self.set_delays(source) - self.set_init_perms(source) + if source != "bypass": + # "bypass" val is used when unpackaging over connection + assert source is None or source is 1 + self.source = source + self.set_delays(source) + self.set_init_perms(source) def set_delays(self, source): """Delay writing permissions and times where appropriate""" @@ -59,13 +64,14 @@ class DSRPath(RPath): # Now get atime right away if possible if self.data.has_key('atime'): self.newatime = self.data['atime'] else: self.newatime = None + else: self.delay_atime = None if source: self.delay_mtime = None # we'll never change mtime of source file else: self.delay_mtime = 1 # Save mtime now for a dir, because it might inadvertantly change - if self.isdir(): self.newmtime = self.getmtime() + if self.isdir(): self.newmtime = self.data['mtime'] else: self.newmtime = None def set_init_perms(self, source): @@ -75,26 +81,30 @@ class DSRPath(RPath): self.chmod_bypass(0400) else: self.warn("No read permissions") elif self.isdir(): - if source and (not self.readable() or self.executable()): + if source and (not self.readable() or not self.executable()): if Globals.change_source_perms and self.isowner(): self.chmod_bypass(0500) - else: warn("No read or exec permission") + else: self.warn("No read or exec permission") elif not source and not self.hasfullperms(): self.chmod_bypass(0700) def warn(self, err): Log("Received error '%s' when dealing with file %s, skipping..." % (err, self.path), 1) - raise DSRPermError(self.path) + raise DSRPPermError(self.path) def __getstate__(self): """Return picklable state. See RPath __getstate__.""" assert self.conn is Globals.local_connection # Can't pickle a conn + return self.getstatedict() + + def getstatedict(self): + """Return dictionary containing the attributes we can save""" pickle_dict = {} for attrib in ['index', 'data', 'delay_perms', 'newperms', 'delay_atime', 'newatime', 'delay_mtime', 'newmtime', - 'path', 'base']: + 'path', 'base', 'source']: if self.__dict__.has_key(attrib): pickle_dict[attrib] = self.__dict__[attrib] return pickle_dict @@ -110,10 +120,17 @@ class DSRPath(RPath): if self.delay_perms: self.newperms = self.data['perms'] = permissions else: RPath.chmod(self, permissions) + def getperms(self): + """Return dsrp's intended permissions""" + if self.delay_perms and self.newperms is not None: + return self.newperms + else: return self.data['perms'] + def chmod_bypass(self, permissions): """Change permissions without updating the data dictionary""" self.delay_perms = 1 if self.newperms is None: self.newperms = self.getperms() + Log("DSRP: Perm bypass %s to %o" % (self.path, permissions), 8) self.conn.os.chmod(self.path, permissions) def settime(self, accesstime, modtime): @@ -129,11 +146,25 @@ class DSRPath(RPath): if self.delay_mtime: self.newmtime = self.data['mtime'] = modtime else: RPath.setmtime(self, modtime) + def getmtime(self): + """Return dsrp's intended modification time""" + if self.delay_mtime and self.newmtime is not None: + return self.newmtime + else: return self.data['mtime'] + + def getatime(self): + """Return dsrp's intended access time""" + if self.delay_atime and self.newatime is not None: + return self.newatime + else: return self.data['atime'] + def write_changes(self): """Write saved up permission/time changes""" if not self.lstat(): return # File has been deleted in meantime if self.delay_perms and self.newperms is not None: + Log("Finalizing permissions of dsrp %s to %s" % + (self.path, self.newperms), 8) RPath.chmod(self, self.newperms) do_atime = self.delay_atime and self.newatime is not None @@ -145,6 +176,19 @@ class DSRPath(RPath): elif not do_atime and do_mtime: RPath.setmtime(self, self.newmtime) + def newpath(self, newpath, index = ()): + """Return similar DSRPath but with new path""" + return self.__class__(self.source, self.conn, newpath, index) + + def append(self, ext): + """Return similar DSRPath with new extension""" + return self.__class__(self.source, self.conn, self.base, + self.index + (ext,)) + + def new_index(self, index): + """Return similar DSRPath with new index""" + return self.__class__(self.source, self.conn, self.base, index) + class DestructiveSteppingFinalizer(IterTreeReducer): """Finalizer that can work on an iterator of dsrpaths @@ -155,11 +199,12 @@ class DestructiveSteppingFinalizer(IterTreeReducer): coming back to it. """ + dsrpath = None def start_process(self, index, dsrpath): self.dsrpath = dsrpath def end_process(self): - self.dsrpath.write_changes() + if self.dsrpath: self.dsrpath.write_changes() diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py index d0bc743..36ba55a 100644 --- a/rdiff-backup/rdiff_backup/highlevel.py +++ b/rdiff-backup/rdiff_backup/highlevel.py @@ -24,12 +24,14 @@ class HighLevel: accompanying diagram. """ - def Mirror(src_rpath, dest_rpath, checkpoint = 1, session_info = None): + def Mirror(src_rpath, dest_rpath, checkpoint = 1, + session_info = None, write_finaldata = 1): """Turn dest_rpath into a copy of src_rpath Checkpoint true means to checkpoint periodically, otherwise not. If session_info is given, try to resume Mirroring from - that point. + that point. If write_finaldata is true, save extra data files + like hardlink_data. If it is false, make a complete mirror. """ SourceS = src_rpath.conn.HLSourceStruct @@ -40,7 +42,8 @@ class HighLevel: src_init_dsiter = SourceS.split_initial_dsiter() dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter) diffiter = SourceS.get_diffs_and_finalize(dest_sigiter) - DestS.patch_and_finalize(dest_rpath, diffiter, checkpoint) + DestS.patch_and_finalize(dest_rpath, diffiter, + checkpoint, write_finaldata) dest_rpath.setdata() @@ -61,24 +64,6 @@ class HighLevel: dest_rpath.setdata() inc_rpath.setdata() - def Restore(rest_time, mirror_base, rel_index, baseinc_tup, target_base): - """Like Restore.RestoreRecursive but check arguments""" - if (Globals.preserve_hardlinks != 0 and - Hardlink.retrieve_final(rest_time)): - Log("Hard link information found, attempting to preserve " - "hard links.", 4) - SetConnections.UpdateGlobal('preserve_hardlinks', 1) - else: SetConnections.UpdateGlobal('preserve_hardlinks', None) - - if not isinstance(target_base, DSRPath): - target_base = DSRPath(target_base.conn, target_base.base, - target_base.index, target_base.data) - if not isinstance(mirror_base, DSRPath): - mirror_base = DSRPath(mirror_base.conn, mirror_base.base, - mirror_base.index, mirror_base.data) - Restore.RestoreRecursive(rest_time, mirror_base, rel_index, - baseinc_tup, target_base) - MakeStatic(HighLevel) @@ -164,7 +149,7 @@ class HLDestinationStruct: def compare(src_rorp, dest_dsrp): """Return dest_dsrp if they are different, None if the same""" if not dest_dsrp: - dest_dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index) + dest_dsrp = cls.get_dsrp(baserp, src_rorp.index) if dest_dsrp.lstat(): Log("Warning: Found unexpected destination file %s, " "not processing it." % dest_dsrp.path, 2) @@ -203,8 +188,9 @@ class HLDestinationStruct: def get_dsrp(cls, dest_rpath, index): """Return initialized dsrp based on dest_rpath with given index""" - return DSRPath(source = None, dest_rpath.conn, - dest_rpath.base, index) + dsrp = DSRPath(None, dest_rpath.conn, dest_rpath.base, index) + if Globals.quoting_enabled: dsrp.quote_path() + return dsrp def get_finalizer(cls): """Return finalizer, starting from session info if necessary""" @@ -216,9 +202,13 @@ class HLDestinationStruct: """Return ITR, starting from state if necessary""" if cls._session_info and cls._session_info.ITR: return cls._session_info.ITR - else: return IncrementITR(inc_rpath) + else: + iitr = IncrementITR(inc_rpath) + iitr.override_changed() + return iitr - def patch_and_finalize(cls, dest_rpath, diffs, checkpoint = 1): + def patch_and_finalize(cls, dest_rpath, diffs, + checkpoint = 1, write_finaldata = 1): """Apply diffs and finalize""" collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) finalizer = cls.get_finalizer() @@ -242,7 +232,7 @@ class HLDestinationStruct: if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp) except: cls.handle_last_error(dsrp, finalizer) finalizer.Finish() - if Globals.preserve_hardlinks and Globals.rbdir: + if Globals.preserve_hardlinks and write_finaldata: Hardlink.final_writedata() if checkpoint: SaveState.checkpoint_remove() @@ -300,8 +290,7 @@ class HLDestinationStruct: Log.exception(1) if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1) else: SaveState.checkpoint_mirror(finalizer, dsrp, 1) - if Globals.preserve_hardlinks: - Hardlink.final_checkpoint(Globals.rbdir) + if Globals.preserve_hardlinks: Hardlink.final_checkpoint(Globals.rbdir) SaveState.touch_last_file_definitive() raise diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py index 446806b..b28b315 100644 --- a/rdiff-backup/rdiff_backup/increment.py +++ b/rdiff-backup/rdiff_backup/increment.py @@ -1,4 +1,4 @@ -execfile("selection.py") +execfile("filename_mapping.py") ####################################################################### # @@ -85,10 +85,12 @@ class Inc: """Get new increment rp with given time suffix""" addtostr = lambda s: "%s.%s.%s" % (s, timestr, typestr) if rp.index: - return rp.__class__(rp.conn, rp.base, rp.index[:-1] + - (addtostr(rp.index[-1]),)) - else: return rp.__class__(rp.conn, addtostr(rp.base), rp.index) - + incrp = rp.__class__(rp.conn, rp.base, rp.index[:-1] + + (addtostr(rp.index[-1]),)) + else: incrp = rp.__class__(rp.conn, addtostr(rp.base), rp.index) + if Globals.quoting_enabled: incrp.quote_path() + return incrp + inctime = 0 while 1: inctime = Resume.FindTime(rp.index, inctime) @@ -123,7 +125,7 @@ class IncrementITR(IterTreeReducer): def __init__(self, inc_rpath): """Set inc_rpath, an rpath of the base of the tree""" self.inc_rpath = inc_rpath - IterTreeReducer.__init__(inc_rpath) + IterTreeReducer.__init__(self, inc_rpath) def start_process(self, index, diff_rorp, dsrp): """Initial processing of file @@ -133,11 +135,21 @@ class IncrementITR(IterTreeReducer): """ incpref = self.inc_rpath.new_index(index) + if Globals.quoting_enabled: incpref.quote_path() if dsrp.isdir(): self.init_dir(dsrp, diff_rorp, incpref) self.setvals(diff_rorp, dsrp, incpref) else: self.init_non_dir(dsrp, diff_rorp, incpref) + def override_changed(self): + """Set changed flag to true + + This is used only at the top level of a backup, to make sure + that a marker is created recording every backup session. + + """ + self.changed = 1 + def setvals(self, diff_rorp, dsrp, incpref): """Record given values in state dict since in directory @@ -162,7 +174,7 @@ class IncrementITR(IterTreeReducer): """ if not (incpref.lstat() and incpref.isdir()): incpref.mkdir() if diff_rorp and diff_rorp.isreg() and diff_rorp.file: - tf = TempFileManager(dsrp) + tf = TempFileManager.new(dsrp) RPathStatic.copy_with_attribs(diff_rorp, tf) tf.set_attached_filetype(diff_rorp.get_attached_filetype()) self.directory_replacement = tf @@ -170,7 +182,7 @@ class IncrementITR(IterTreeReducer): def init_non_dir(self, dsrp, diff_rorp, incpref): """Process a non directory file (initial pass)""" if not diff_rorp: return # no diff, so no change necessary - if diff_rorp.isreg and (dsrp.isreg() or diff_rorp.isflaglinked()): + if diff_rorp.isreg() and (dsrp.isreg() or diff_rorp.isflaglinked()): tf = TempFileManager.new(dsrp) def init_thunk(): if diff_rorp.isflaglinked(): @@ -180,8 +192,8 @@ class IncrementITR(IterTreeReducer): Inc.Increment_action(tf, dsrp, incpref).execute() Robust.make_tf_robustaction(init_thunk, (tf,), (dsrp,)).execute() else: - Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incref), - RORPIter.patchonce_action(none, dsrp, diff_rorp)] + Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incpref), + RORPIter.patchonce_action(None, dsrp, diff_rorp)] ).execute() self.changed = 1 @@ -207,89 +219,3 @@ class IncrementITR(IterTreeReducer): - - - def make_patch_increment_ITR(inc_rpath, initial_state = None): - """Return IterTreeReducer that patches and increments""" - def base_init(indexed_tuple): - """Patch if appropriate, return (a,b) tuple - - a is true if found directory and thus didn't take action - - if a is false, b is true if some changes were made - - if a is true, b is the rp of a temporary file used to hold - the diff_rorp's data (for dir -> normal file change), and - false if none was necessary. - - """ - diff_rorp, dsrp = indexed_tuple - incpref = inc_rpath.new_index(indexed_tuple.index) - if dsrp.isdir(): return init_dir(dsrp, diff_rorp, incpref) - else: return init_non_dir(dsrp, diff_rorp, incpref) - - def init_dir(dsrp, diff_rorp, incpref): - """Initial processing of a directory - - Make the corresponding directory right away, but wait - until the end to write the replacement. However, if the - diff_rorp contains data, we must write it locally before - continuing, or else that data will be lost in the stream. - - """ - if not (incpref.lstat() and incpref.isdir()): incpref.mkdir() - if diff_rorp and diff_rorp.isreg() and diff_rorp.file: - tf = TempFileManager.new(dsrp) - RPathStatic.copy_with_attribs(diff_rorp, tf) - tf.set_attached_filetype(diff_rorp.get_attached_filetype()) - return (1, tf) - else: return (1, None) - - def init_non_dir(dsrp, diff_rorp, incpref): - """Initial processing of non-directory - - If a reverse diff is called for it is generated by apply - the forwards diff first on a temporary file. - - """ - if diff_rorp: - if diff_rorp.isreg() and (dsrp.isreg() or - diff_rorp.isflaglinked()): - tf = TempFileManager.new(dsrp) - def init_thunk(): - if diff_rorp.isflaglinked(): - Hardlink.link_rp(diff_rorp, tf, dsrp) - else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp, - tf).execute() - Inc.Increment_action(tf, dsrp, incpref).execute() - Robust.make_tf_robustaction(init_thunk, (tf,), - (dsrp,)).execute() - else: - Robust.chain([Inc.Increment_action(diff_rorp, dsrp, - incpref), - RORPIter.patchonce_action( - None, dsrp, diff_rorp)]).execute() - return (None, 1) - return (None, None) - - def base_final(base_tuple, base_init_tuple, changed): - """Patch directory if not done, return true iff made change""" - if base_init_tuple[0]: # was directory - diff_rorp, dsrp = base_tuple - if changed or diff_rorp: - if base_init_tuple[1]: diff_rorp = base_init_tuple[1] - Inc.Increment(diff_rorp, dsrp, - inc_rpath.new_index(base_tuple.index)) - if diff_rorp: - RORPIter.patchonce_action(None, dsrp, - diff_rorp).execute() - if isinstance(diff_rorp, TempFile): diff_rorp.delete() - return 1 - return None - else: # changed iff base_init_tuple says it was - return base_init_tuple[1] - - return IterTreeReducer(base_init, lambda x,y: x or y, None, - base_final, initial_state) - - diff --git a/rdiff-backup/rdiff_backup/lazy.py b/rdiff-backup/rdiff_backup/lazy.py index 80cfa95..1bb2e2c 100644 --- a/rdiff-backup/rdiff_backup/lazy.py +++ b/rdiff-backup/rdiff_backup/lazy.py @@ -260,7 +260,7 @@ class IterTreeReducer: """ index = args[0] - assert type(index) is types.TupleType + assert type(index) is types.TupleType, type(index) if self.index is None: self.start_process(*args) diff --git a/rdiff-backup/rdiff_backup/manage.py b/rdiff-backup/rdiff_backup/manage.py index c0f4a85..0c08872 100644 --- a/rdiff-backup/rdiff_backup/manage.py +++ b/rdiff-backup/rdiff_backup/manage.py @@ -12,37 +12,53 @@ class Manage: """Return Increments objects given the rdiff-backup data directory""" return map(IncObj, Manage.find_incrps_with_base(datadir, "increments")) - def find_incrps_with_base(dir_rp, basename): - """Return list of incfiles with given basename in dir_rp""" - rps = map(dir_rp.append, dir_rp.listdir()) - incrps = filter(RPath.isincfile, rps) - result = filter(lambda rp: rp.getincbase_str() == basename, incrps) - Log("find_incrps_with_base: found %d incs" % len(result), 6) - return result + def get_file_type(rp): + """Returns one of "regular", "directory", "missing", or "special".""" + if not rp.lstat(): return "missing" + elif rp.isdir(): return "directory" + elif rp.isreg(): return "regular" + else: return "special" - def describe_root_incs(datadir): + def get_inc_type(inc): + """Return file type increment represents""" + assert inc.isincfile() + type = inc.getinctype() + if type == "dir": return "directory" + elif type == "diff": return "regular" + elif type == "missing": return "missing" + elif type == "snapshot": return Manage.get_file_type(inc) + else: assert None, "Unknown type %s" % (type,) + + def describe_incs_parsable(incs, mirror_time, mirrorrp): + """Return a string parsable by computer describing the increments + + Each line is a time in seconds of the increment, and then the + type of the file. It will be sorted oldest to newest. For example: + + 10000 regular + 20000 directory + 30000 special + 40000 missing + 50000 regular <- last will be the current mirror + + """ + incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs] + incpairs.sort() + result = ["%s %s" % (time, Manage.get_inc_type(inc)) + for time, inc in incpairs] + result.append("%s %s" % (mirror_time, Manage.get_file_type(mirrorrp))) + return "\n".join(result) + + def describe_incs_human(incs, mirror_time, mirrorrp): """Return a string describing all the the root increments""" - result = [] - currentrps = Manage.find_incrps_with_base(datadir, "current_mirror") - if not currentrps: - Log("Warning: no current mirror marker found", 1) - elif len(currentrps) > 1: - Log("Warning: multiple mirror markers found", 1) - for rp in currentrps: - result.append("Found mirror marker %s" % rp.path) - result.append("Indicating latest mirror taken at %s" % - Time.stringtopretty(rp.getinctime())) - result.append("---------------------------------------------" - "-------------") - - # Sort so they are in reverse order by time - time_w_incobjs = map(lambda io: (-io.time, io), - Manage.get_incobjs(datadir)) - time_w_incobjs.sort() - incobjs = map(lambda x: x[1], time_w_incobjs) - result.append("Found %d increments:" % len(incobjs)) - result.append("\n------------------------------------------\n".join( - map(IncObj.full_description, incobjs))) + incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs] + incpairs.sort() + + result = ["Found %d increments:" % len(incpairs)] + for time, inc in incpairs: + result.append(" %s %s" % + (inc.dirsplit()[1], Time.timetopretty(time))) + result.append("Current mirror: %s" % Time.timetopretty(mirror_time)) return "\n".join(result) def delete_earlier_than(baserp, time): @@ -53,6 +69,11 @@ class Manage: rdiff-backup-data directory should be the root of the tree. """ + baserp.conn.Manage.delete_earlier_than_local(baserp, time) + + def delete_earlier_than_local(baserp, time): + """Like delete_earlier_than, but run on local connection for speed""" + assert baserp.conn is Globals.local_connection def yield_files(rp): yield rp if rp.isdir(): diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py index dcba7f3..0faa9b2 100644 --- a/rdiff-backup/rdiff_backup/restore.py +++ b/rdiff-backup/rdiff_backup/restore.py @@ -24,32 +24,78 @@ class Restore: same index as mirror. """ - if not isinstance(mirror, DSRPath): - mirror = DSRPath(source = 1, mirror) - if not isinstance(target, DSRPath): - target = DSRPath(source = None, target) + if not isinstance(mirror, DSRPath): mirror = DSRPath(1, mirror) + if not isinstance(target, DSRPath): target = DSRPath(None, target) + + mirror_time = Restore.get_mirror_time() + rest_time = Restore.get_rest_time(rest_time) + inc_list = Restore.get_inclist(inc_rpath) + rid = RestoreIncrementData(inc_rpath.index, inc_rpath, inc_list) + rid.sortincseq(rest_time, mirror_time) + Restore.check_hardlinks(rest_time) + Restore.restore_recursive(inc_rpath.index, mirror, rid, target, + rest_time, mirror_time) + + def get_mirror_time(): + """Return the time (in seconds) of latest mirror""" + current_mirror_incs = \ + Restore.get_inclist(Globals.rbdir.append("current_mirror")) + if not current_mirror_incs: + Log.FatalError("Could not get time of current mirror") + elif len(current_mirror_incs) > 1: + Log("Warning, two different dates for current mirror found", 2) + return Time.stringtotime(current_mirror_incs[0].getinctime()) + + def get_rest_time(old_rest_time): + """If old_rest_time is between two increments, return older time + + There is a slightly tricky reason for doing this: The rest of + the code just ignores increments that are older than + rest_time. But sometimes we want to consider the very next + increment older than rest time, because rest_time will be + between two increments, and what was actually on the mirror + side will correspond to the older one. + + So here we assume all rdiff-backup events were recorded in + "increments" increments, and if its in-between we pick the + older one here. + """ + base_incs = Restore.get_inclist(Globals.rbdir.append("increments")) + if not base_incs: return old_rest_time + inctimes = [Time.stringtotime(inc.getinctime()) for inc in base_incs] + return max(filter(lambda time: time <= old_rest_time, inctimes)) + + def get_inclist(inc_rpath): + """Returns increments with given base""" dirname, basename = inc_rpath.dirsplit() parent_dir = RPath(inc_rpath.conn, dirname, ()) index = inc_rpath.index - if inc_rpath.index: + if index: get_inc_ext = lambda filename: \ RPath(inc_rpath.conn, inc_rpath.base, inc_rpath.index[:-1] + (filename,)) else: get_inc_ext = lambda filename: \ - RPath(inc_rpath.conn, os.join(dirname, filename)) + RPath(inc_rpath.conn, os.path.join(dirname, filename)) inc_list = [] for filename in parent_dir.listdir(): inc = get_inc_ext(filename) - if inc.getincbase_str() == basename: inc_list.append(inc) - - rid = RestoreIncrementData(index, inc_rpath, inc_list) - rid.sortincseq(rest_time) - Restore.restore_recursive(index, mirror, rid, target, rest_time) - - def restore_recursive(index, mirror, rid, target, time): + if inc.isincfile() and inc.getincbase_str() == basename: + inc_list.append(inc) + return inc_list + + def check_hardlinks(rest_time): + """Check for hard links and enable hard link support if found""" + if (Globals.preserve_hardlinks != 0 and + Hardlink.retrieve_final(rest_time)): + Log("Hard link information found, attempting to preserve " + "hard links.", 5) + SetConnections.UpdateGlobal('preserve_hardlinks', 1) + else: SetConnections.UpdateGlobal('preserve_hardlinks', None) + + def restore_recursive(index, mirror, rid, target, time, mirror_time): """Recursive restore function. rid is a RestoreIncrementData object whose inclist is already @@ -66,14 +112,15 @@ class Restore: mirror_finalizer = DestructiveSteppingFinalizer() target_finalizer = DestructiveSteppingFinalizer() - for rcd in Restore.yield_rcds(rid.index, mirror, rid, target, time): + for rcd in Restore.yield_rcds(rid.index, mirror, rid, + target, time, mirror_time): rcd.RestoreFile() - if rcd.mirror: mirror_finalizer(rcd.mirror) - target_finalizer(rcd.target) + if rcd.mirror: mirror_finalizer(rcd.index, rcd.mirror) + target_finalizer(rcd.target.index, rcd.target) target_finalizer.Finish() mirror_finalizer.Finish() - def yield_rcds(index, mirrorrp, rid, target, rest_time): + def yield_rcds(index, mirrorrp, rid, target, rest_time, mirror_time): """Iterate RestoreCombinedData objects starting with given args rid is a RestoreCombinedData object. target is an rpath where @@ -91,9 +138,10 @@ class Restore: mirrorrp = None rcd = RestoreCombinedData(rid, mirrorrp, target) - if mirrorrp and mirrorrp.isdir() or rid and rid.inc_rpath.isdir(): + if mirrorrp and mirrorrp.isdir() or \ + rid and rid.inc_rpath and rid.inc_rpath.isdir(): sub_rcds = Restore.yield_sub_rcds(index, mirrorrp, rid, - target, rest_time) + target, rest_time, mirror_time) else: sub_rcds = None if select_result == 1: @@ -108,35 +156,39 @@ class Restore: yield first for sub_rcd in sub_rcds: yield sub_rcd - def yield_collated_tuples_dir(index, mirrorrp, rid, target, rest_time): + def yield_sub_rcds(index, mirrorrp, rid, target, rest_time, mirror_time): """Yield collated tuples from inside given args""" - if not Restore.check_dir_exists(mirrorrp, inc_tup): return + if not Restore.check_dir_exists(mirrorrp, rid): return mirror_iter = Restore.yield_mirrorrps(mirrorrp) - rid_iter = Restore.get_rids(rid, rest_time) + rid_iter = Restore.yield_rids(rid, rest_time, mirror_time) for indexed_tup in RORPIter.CollateIterators(mirror_iter, rid_iter): index = indexed_tup.index new_mirrorrp, new_rid = indexed_tup - for rcd in Restore.yield_collated_tuples(index, new_mirrorrp, - new_rid, target.new_index(index), rest_time): + for rcd in Restore.yield_rcds(index, new_mirrorrp, + new_rid, target.append(index[-1]), rest_time, mirror_time): yield rcd - def check_dir_exists(mirrorrp, inc_tuple): + def check_dir_exists(mirrorrp, rid): """Return true if target should be a directory""" - if inc_tuple and inc_tuple[1]: + if rid and rid.inc_list: # Incs say dir if last (earliest) one is a dir increment - return inc_tuple[1][-1].getinctype() == "dir" + return rid.inc_list[-1].getinctype() == "dir" elif mirrorrp: return mirrorrp.isdir() # if no incs, copy mirror else: return None def yield_mirrorrps(mirrorrp): """Yield mirrorrps underneath given mirrorrp""" if mirrorrp and mirrorrp.isdir(): - dirlist = mirrorrp.listdir() - dirlist.sort() - for filename in dirlist: yield mirrorrp.append(filename) - - def yield_rids(rid, rest_time): + if Globals.quoting_enabled: + for rp in FilenameMapping.get_quoted_dir_children(mirrorrp): + yield rp + else: + dirlist = mirrorrp.listdir() + dirlist.sort() + for filename in dirlist: yield mirrorrp.append(filename) + + def yield_rids(rid, rest_time, mirror_time): """Yield RestoreIncrementData objects within given rid dir If the rid doesn't correspond to a directory, don't yield any @@ -148,16 +200,19 @@ class Restore: if not rid or not rid.inc_rpath or not rid.inc_rpath.isdir(): return rid_dict = {} # dictionary of basenames:rids dirlist = rid.inc_rpath.listdir() + if Globals.quoting_enabled: + dirlist = [FilenameMapping.unquote(fn) for fn in dirlist] def affirm_dict_indexed(basename): """Make sure the rid dictionary has given basename as key""" - if not inc_list_dict.has_key(basename): + if not rid_dict.has_key(basename): rid_dict[basename] = RestoreIncrementData( rid.index + (basename,), None, []) # init with empty rid def add_to_dict(filename): """Add filename to the inc tuple dictionary""" rp = rid.inc_rpath.append(filename) + if Globals.quoting_enabled: rp.quote_path() if rp.isincfile(): basename = rp.getincbase_str() affirm_dict_indexed(basename) @@ -167,14 +222,14 @@ class Restore: rid_dict[filename].inc_rpath = rp for filename in dirlist: add_to_dict(filename) - keys = inc_list_dict.keys() + keys = rid_dict.keys() keys.sort() # sortincseq now to avoid descending .missing directories later for key in keys: rid = rid_dict[key] if rid.inc_rpath or rid.inc_list: - rid.sortincseq(rest_time) + rid.sortincseq(rest_time, mirror_time) yield rid MakeStatic(Restore) @@ -192,26 +247,36 @@ class RestoreIncrementData: self.inc_rpath = inc_rpath self.inc_list = inc_list - def sortincseq(self, rest_time): + def sortincseq(self, rest_time, mirror_time): """Sort self.inc_list sequence, throwing away irrelevant increments""" - incpairs = map(lambda rp: (Time.stringtotime(rp.getinctime()), rp), - self.inc_list) - # Only consider increments at or after the time being restored - incpairs = filter(lambda pair: pair[0] >= rest_time, incpairs) + if not self.inc_list or rest_time >= mirror_time: + self.inc_list = [] + return - # Now throw away older unnecessary increments - incpairs.sort() + newer_incs = self.get_newer_incs(rest_time, mirror_time) i = 0 - while(i < len(incpairs)): + while(i < len(newer_incs)): # Only diff type increments require later versions - if incpairs[i][1].getinctype() != "diff": break + if newer_incs[i].getinctype() != "diff": break i = i+1 - incpairs = incpairs[:i+1] + self.inc_list = newer_incs[:i+1] + self.inc_list.reverse() # return in reversed order (latest first) + + def get_newer_incs(self, rest_time, mirror_time): + """Return list of newer incs sorted by time (increasing) - # Return increments in reversed order (latest first) - incpairs.reverse() - self.inc_list = map(lambda pair: pair[1], incpairs) + Also discard increments older than rest_time (rest_time we are + assuming is the exact time rdiff-backup was run, so no need to + consider the next oldest increment or any of that) + """ + incpairs = [] + for inc in self.inc_list: + time = Time.stringtotime(inc.getinctime()) + if time >= rest_time: incpairs.append((time, inc)) + incpairs.sort() + return [pair[1] for pair in incpairs] + class RestoreCombinedData: """Combine index information from increment and mirror directories @@ -235,9 +300,12 @@ class RestoreCombinedData: if mirror: self.mirror = mirror assert mirror.index == self.index + else: self.mirror = None elif mirror: self.index = mirror.index self.mirror = mirror + self.inc_list = [] + self.inc_rpath = None else: assert None, "neither rid nor mirror given" self.target = target @@ -249,15 +317,15 @@ class RestoreCombinedData: if self.restore_hardlink(): return - if not inclist or inclist[0].getinctype() == "diff": + if not self.inc_list or self.inc_list[0].getinctype() == "diff": assert self.mirror and self.mirror.lstat(), \ "No base to go with incs for %s" % self.target.path RPath.copy_with_attribs(self.mirror, self.target) for inc in self.inc_list: self.applyinc(inc, self.target) - def log(self) + def log(self): """Log current restore action""" - inc_string = ','.join(map(lambda x: x.path, self.inc_list)) + inc_string = ','.join([inc.path for inc in self.inc_list]) Log("Restoring %s with increments %s to %s" % (self.mirror and self.mirror.path, inc_string, self.target.path), 5) @@ -266,7 +334,7 @@ class RestoreCombinedData: """Hard link target and return true if hard linking appropriate""" if (Globals.preserve_hardlinks and Hardlink.restore_link(self.index, self.target)): - RPath.copy_attribs(self.inc_list and inc_list[-1] or + RPath.copy_attribs(self.inc_list and self.inc_list[-1] or self.mirror, self.target) return 1 return None diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py index 22f35b9..a71eabc 100644 --- a/rdiff-backup/rdiff_backup/robust.py +++ b/rdiff-backup/rdiff_backup/robust.py @@ -252,7 +252,7 @@ class TempFile(RPath): if self.isdir() and not rp_dest.isdir(): # Cannot move a directory directly over another file rp_dest.delete() - if (isinstance(rp_dest, DSRPath) and rp_dest.perms_delayed + if (isinstance(rp_dest, DSRPath) and rp_dest.delay_perms and not self.hasfullperms()): # If we are moving to a delayed perm directory, delay # permission change on destination. @@ -531,7 +531,7 @@ class Resume: Log("Last backup dated %s was aborted, but we aren't " "resuming it." % Time.timetopretty(si.time), 2) return None - assert 0 + assert None MakeClass(Resume) diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py index 0089bf6..110b89f 100644 --- a/rdiff-backup/rdiff_backup/rpath.py +++ b/rdiff-backup/rdiff_backup/rpath.py @@ -168,6 +168,17 @@ class RPathStatic: rp_dest.data = rp_source.data rp_source.data = {'type': None} + # If we are moving to a DSRPath, assume that the current times + # are the intended ones. We need to save them now in case + # they are changed later. + if isinstance(rp_dest, DSRPath): + if rp_dest.delay_mtime: + if 'mtime' in rp_dest.data: + rp_dest.setmtime(rp_dest.data['mtime']) + if rp_dest.delay_atime: + if 'atime' in rp_dest.data: + rp_dest.setatime(rp_dest.data['atime']) + def tupled_lstat(filename): """Like os.lstat, but return only a tuple, or None if os.error @@ -413,7 +424,7 @@ class RPath(RORPath): self.base = base self.path = apply(os.path.join, (base,) + self.index) self.file = None - if data: self.data = data + if data or base is None: self.data = data else: self.setdata() def __str__(self): @@ -493,6 +504,12 @@ class RPath(RORPath): s = self.conn.reval("lambda path: os.lstat(path).st_rdev", self.path) return (s >> 8, s & 0xff) + def quote_path(self): + """Set path from quoted version of index""" + quoted_list = [FilenameMapping.quote(path) for path in self.index] + self.path = apply(os.path.join, [self.base] + quoted_list) + self.setdata() + def chmod(self, permissions): """Wrapper around os.chmod""" self.conn.os.chmod(self.path, permissions) @@ -594,7 +611,8 @@ class RPath(RORPath): if not self.lstat(): return # must have been deleted in meantime elif self.isdir(): itm = RpathDeleter() - for dsrp in Select(self, None).set_iter(): itm(dsrp.index, dsrp) + for dsrp in Select(DSRPath(None, self)).set_iter(): + itm(dsrp.index, dsrp) itm.Finish() else: self.conn.os.unlink(self.path) self.setdata() @@ -616,7 +634,7 @@ class RPath(RORPath): self.path.split("/"))) if self.path[0] == "/": newpath = "/" + newpath elif not newpath: newpath = "." - return self.__class__(self.conn, newpath, ()) + return self.newpath(newpath) def dirsplit(self): """Returns a tuple of strings (dirname, basename) @@ -635,10 +653,20 @@ class RPath(RORPath): comps = normed.path.split("/") return "/".join(comps[:-1]), comps[-1] + def newpath(self, newpath, index = ()): + """Return new RPath with the same connection but different path""" + return self.__class__(self.conn, newpath, index) + def append(self, ext): """Return new RPath with same connection by adjoing ext""" return self.__class__(self.conn, self.base, self.index + (ext,)) + def append_path(self, ext, new_index = ()): + """Like append, but add ext to path instead of to index""" + assert not self.index # doesn't make sense if index isn't () + return self.__class__(self.conn, os.path.join(self.base, ext), + new_index) + def new_index(self, index): """Return similar RPath but with new index""" return self.__class__(self.conn, self.base, index) diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py index aaa8639..cae6db3 100644 --- a/rdiff-backup/rdiff_backup/selection.py +++ b/rdiff-backup/rdiff_backup/selection.py @@ -60,21 +60,20 @@ class Select: # This re should not match normal filenames, but usually just globs glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S) - def __init__(self, rpath, source): - """DSRPIterator initializer. + def __init__(self, dsrpath, quoted_filenames = None): + """DSRPIterator initializer. dsrp is the root directory - rpath is the root dir. Source is true if rpath is the root of - the source directory, and false for the mirror directory + When files have quoted characters in them, quoted_filenames + should be true. Then RPath's index will be the unquoted + version. """ - assert isinstance(rpath, RPath) + assert isinstance(dsrpath, DSRPath) self.selection_functions = [] - self.source = source - if isinstance(rpath, DSRPath): self.dsrpath = rpath - else: self.dsrpath = DSRPath(rpath.conn, rpath.base, - rpath.index, rpath.data) + self.dsrpath = dsrpath self.prefix = self.dsrpath.path - + self.quoting_on = Globals.quoting_enabled and quoted_filenames + def set_iter(self, starting_index = None, sel_func = None): """Initialize more variables, get ready to iterate @@ -106,7 +105,7 @@ class Select: """ s = sel_func(dsrpath) - if s === 0: return + if s == 0: return elif s == 1: # File is included yield dsrpath if dsrpath.isdir(): @@ -122,11 +121,15 @@ class Select: def iterate_in_dir(self, dsrpath, rec_func, sel_func): """Iterate the dsrps in directory dsrpath.""" - dir_listing = dsrpath.listdir() - dir_listing.sort() - for filename in dir_listing: - for dsrp in rec_func(dsrpath.append(filename), rec_func, sel_func): - yield dsrp + if self.quoting_on: + for subdir in FilenameMapping.get_quoted_dir_children(dsrpath): + for dsrp in rec_func(subdir, rec_func, sel_func): yield dsrp + else: + dir_listing = dsrpath.listdir() + dir_listing.sort() + for filename in dir_listing: + for dsrp in rec_func(dsrpath.append(filename), + rec_func, sel_func): yield dsrp def iterate_starting_from(self, dsrpath, rec_func, sel_func): """Like Iterate, but only yield indicies > self.starting_index""" diff --git a/rdiff-backup/src/Make b/rdiff-backup/src/Make index 6e66656..b78ed95 100755 --- a/rdiff-backup/src/Make +++ b/rdiff-backup/src/Make @@ -23,9 +23,9 @@ def mystrip(filename): files = ["globals.py", "static.py", "lazy.py", "log.py", "ttime.py", "iterfile.py", "rdiff.py", "connection.py", "rpath.py", "hardlink.py", "robust.py", "rorpiter.py", - "destructive_stepping.py", "selection.py", "increment.py", - "restore.py", "manage.py", "highlevel.py", - "setconnections.py", "main.py"] + "destructive_stepping.py", "selection.py", + "filename_mapping.py", "increment.py", "restore.py", + "manage.py", "highlevel.py", "setconnections.py", "main.py"] os.system("cp header.py rdiff-backup") diff --git a/rdiff-backup/src/connection.py b/rdiff-backup/src/connection.py index 9842480..57d2fa5 100644 --- a/rdiff-backup/src/connection.py +++ b/rdiff-backup/src/connection.py @@ -92,6 +92,7 @@ class LowLevelPipeConnection(Connection): b - string q - quit signal t - TempFile + d - DSRPath R - RPath r - RORPath only c - PipeConnection object @@ -118,6 +119,7 @@ class LowLevelPipeConnection(Connection): if type(obj) is types.StringType: self._putbuf(obj, req_num) elif isinstance(obj, Connection): self._putconn(obj, req_num) elif isinstance(obj, TempFile): self._puttempfile(obj, req_num) + elif isinstance(obj, DSRPath): self._putdsrpath(obj, req_num) elif isinstance(obj, RPath): self._putrpath(obj, req_num) elif isinstance(obj, RORPath): self._putrorpath(obj, req_num) elif ((hasattr(obj, "read") or hasattr(obj, "write")) @@ -148,6 +150,11 @@ class LowLevelPipeConnection(Connection): tempfile.index, tempfile.data) self._write("t", cPickle.dumps(tf_repr, 1), req_num) + def _putdsrpath(self, dsrpath, req_num): + """Put DSRPath into pipe. See _putrpath""" + dsrpath_repr = (dsrpath.conn.conn_number, dsrpath.getstatedict()) + self._write("d", cPickle.dumps(dsrpath_repr, 1), req_num) + def _putrpath(self, rpath, req_num): """Put an rpath into the pipe @@ -219,23 +226,22 @@ class LowLevelPipeConnection(Connection): ord(header_string[1]), self._s2l(header_string[2:])) except IndexError: raise ConnectionError() - if format_string == "o": result = cPickle.loads(self._read(length)) - elif format_string == "b": result = self._read(length) - elif format_string == "f": - result = VirtualFile(self, int(self._read(length))) + if format_string == "q": raise ConnectionQuit("Received quit signal") + + data = self._read(length) + if format_string == "o": result = cPickle.loads(data) + elif format_string == "b": result = data + elif format_string == "f": result = VirtualFile(self, int(data)) elif format_string == "i": - result = RORPIter.FromFile(BufferedRead( - VirtualFile(self, int(self._read(length))))) - elif format_string == "t": - result = self._gettempfile(self._read(length)) - elif format_string == "r": - result = self._getrorpath(self._read(length)) - elif format_string == "R": result = self._getrpath(self._read(length)) - elif format_string == "c": - result = Globals.connection_dict[int(self._read(length))] + result = RORPIter.FromFile(BufferedRead(VirtualFile(self, + int(data)))) + elif format_string == "t": result = self._gettempfile(data) + elif format_string == "r": result = self._getrorpath(data) + elif format_string == "R": result = self._getrpath(data) + elif format_string == "d": result = self._getdsrpath(data) else: - assert format_string == "q", header_string - raise ConnectionQuit("Received quit signal") + assert format_string == "c", header_string + result = Globals.connection_dict[int(data)] Log.conn("received", result, req_num) return (req_num, result) @@ -255,6 +261,15 @@ class LowLevelPipeConnection(Connection): conn_number, base, index, data = cPickle.loads(raw_rpath_buf) return RPath(Globals.connection_dict[conn_number], base, index, data) + def _getdsrpath(self, raw_dsrpath_buf): + """Return DSRPath object indicated by buf""" + conn_number, state_dict = cPickle.loads(raw_dsrpath_buf) + empty_dsrp = DSRPath("bypass", Globals.local_connection, None) + empty_dsrp.__setstate__(state_dict) + empty_dsrp.conn = Globals.connection_dict[conn_number] + empty_dsrp.file = None + return empty_dsrp + def _close(self): """Close the pipes associated with the connection""" self.outpipe.close() diff --git a/rdiff-backup/src/destructive_stepping.py b/rdiff-backup/src/destructive_stepping.py index c5e2faa..ff3b42a 100644 --- a/rdiff-backup/src/destructive_stepping.py +++ b/rdiff-backup/src/destructive_stepping.py @@ -1,4 +1,5 @@ from __future__ import generators +import types execfile("rorpiter.py") ####################################################################### @@ -40,13 +41,17 @@ class DSRPath(RPath): otherwise use the same arguments as the RPath initializer. """ - if len(args) == 2 and isinstance(args[0], RPath): + if len(args) == 1 and isinstance(args[0], RPath): rp = args[0] RPath.__init__(self, rp.conn, rp.base, rp.index) else: RPath.__init__(self, *args) - self.set_delays(source) - self.set_init_perms(source) + if source != "bypass": + # "bypass" val is used when unpackaging over connection + assert source is None or source is 1 + self.source = source + self.set_delays(source) + self.set_init_perms(source) def set_delays(self, source): """Delay writing permissions and times where appropriate""" @@ -59,13 +64,14 @@ class DSRPath(RPath): # Now get atime right away if possible if self.data.has_key('atime'): self.newatime = self.data['atime'] else: self.newatime = None + else: self.delay_atime = None if source: self.delay_mtime = None # we'll never change mtime of source file else: self.delay_mtime = 1 # Save mtime now for a dir, because it might inadvertantly change - if self.isdir(): self.newmtime = self.getmtime() + if self.isdir(): self.newmtime = self.data['mtime'] else: self.newmtime = None def set_init_perms(self, source): @@ -75,26 +81,30 @@ class DSRPath(RPath): self.chmod_bypass(0400) else: self.warn("No read permissions") elif self.isdir(): - if source and (not self.readable() or self.executable()): + if source and (not self.readable() or not self.executable()): if Globals.change_source_perms and self.isowner(): self.chmod_bypass(0500) - else: warn("No read or exec permission") + else: self.warn("No read or exec permission") elif not source and not self.hasfullperms(): self.chmod_bypass(0700) def warn(self, err): Log("Received error '%s' when dealing with file %s, skipping..." % (err, self.path), 1) - raise DSRPermError(self.path) + raise DSRPPermError(self.path) def __getstate__(self): """Return picklable state. See RPath __getstate__.""" assert self.conn is Globals.local_connection # Can't pickle a conn + return self.getstatedict() + + def getstatedict(self): + """Return dictionary containing the attributes we can save""" pickle_dict = {} for attrib in ['index', 'data', 'delay_perms', 'newperms', 'delay_atime', 'newatime', 'delay_mtime', 'newmtime', - 'path', 'base']: + 'path', 'base', 'source']: if self.__dict__.has_key(attrib): pickle_dict[attrib] = self.__dict__[attrib] return pickle_dict @@ -110,10 +120,17 @@ class DSRPath(RPath): if self.delay_perms: self.newperms = self.data['perms'] = permissions else: RPath.chmod(self, permissions) + def getperms(self): + """Return dsrp's intended permissions""" + if self.delay_perms and self.newperms is not None: + return self.newperms + else: return self.data['perms'] + def chmod_bypass(self, permissions): """Change permissions without updating the data dictionary""" self.delay_perms = 1 if self.newperms is None: self.newperms = self.getperms() + Log("DSRP: Perm bypass %s to %o" % (self.path, permissions), 8) self.conn.os.chmod(self.path, permissions) def settime(self, accesstime, modtime): @@ -129,11 +146,25 @@ class DSRPath(RPath): if self.delay_mtime: self.newmtime = self.data['mtime'] = modtime else: RPath.setmtime(self, modtime) + def getmtime(self): + """Return dsrp's intended modification time""" + if self.delay_mtime and self.newmtime is not None: + return self.newmtime + else: return self.data['mtime'] + + def getatime(self): + """Return dsrp's intended access time""" + if self.delay_atime and self.newatime is not None: + return self.newatime + else: return self.data['atime'] + def write_changes(self): """Write saved up permission/time changes""" if not self.lstat(): return # File has been deleted in meantime if self.delay_perms and self.newperms is not None: + Log("Finalizing permissions of dsrp %s to %s" % + (self.path, self.newperms), 8) RPath.chmod(self, self.newperms) do_atime = self.delay_atime and self.newatime is not None @@ -145,6 +176,19 @@ class DSRPath(RPath): elif not do_atime and do_mtime: RPath.setmtime(self, self.newmtime) + def newpath(self, newpath, index = ()): + """Return similar DSRPath but with new path""" + return self.__class__(self.source, self.conn, newpath, index) + + def append(self, ext): + """Return similar DSRPath with new extension""" + return self.__class__(self.source, self.conn, self.base, + self.index + (ext,)) + + def new_index(self, index): + """Return similar DSRPath with new index""" + return self.__class__(self.source, self.conn, self.base, index) + class DestructiveSteppingFinalizer(IterTreeReducer): """Finalizer that can work on an iterator of dsrpaths @@ -155,11 +199,12 @@ class DestructiveSteppingFinalizer(IterTreeReducer): coming back to it. """ + dsrpath = None def start_process(self, index, dsrpath): self.dsrpath = dsrpath def end_process(self): - self.dsrpath.write_changes() + if self.dsrpath: self.dsrpath.write_changes() diff --git a/rdiff-backup/src/globals.py b/rdiff-backup/src/globals.py index 710b5e2..2f0b0ca 100644 --- a/rdiff-backup/src/globals.py +++ b/rdiff-backup/src/globals.py @@ -8,7 +8,7 @@ import re, os class Globals: # The current version of rdiff-backup - version = "0.7.3" + version = "0.7.4" # If this is set, use this value in seconds as the current time # instead of reading it from the clock. @@ -108,6 +108,18 @@ class Globals: # under MS windows NT. time_separator = ":" + # quoting_enabled is true if we should quote certain characters in + # filenames on the source side (see FilenameMapping for more + # info). chars_to_quote is a string whose characters should be + # quoted, and quoting_char is the character to quote with. + quoting_enabled = None + chars_to_quote = "" + quoting_char = ';' + + # If true, emit output intended to be easily readable by a + # computer. False means output is intended for humans. + parsable_output = None + # If true, then hardlinks will be preserved to mirror and recorded # in the increments directory. There is also a difference here # between None and 0. When restoring, None or 1 means to preserve @@ -180,12 +192,12 @@ class Globals: else: cls.__dict__[name] = re.compile(re_string) postset_regexp_local = classmethod(postset_regexp_local) - def set_select(cls, source, dsrpath, tuplelist): + def set_select(cls, dsrpath, tuplelist, quote_mode = None): """Initialize select object using tuplelist""" - if source: - cls.select_source = Select(dsrpath, 1) + if dsrpath.source: + cls.select_source = Select(dsrpath, quote_mode) cls.select_source.ParseArgs(tuplelist) else: - cls.select_mirror = Select(dsrpath, None) + cls.select_mirror = Select(dsrpath, quote_mode) cls.select_mirror.ParseArgs(tuplelist) set_select = classmethod(set_select) diff --git a/rdiff-backup/src/hardlink.py b/rdiff-backup/src/hardlink.py index deecd68..7531ad3 100644 --- a/rdiff-backup/src/hardlink.py +++ b/rdiff-backup/src/hardlink.py @@ -28,6 +28,11 @@ class Hardlink: _src_index_indicies = {} _dest_index_indicies = {} + # When a linked file is restored, its path is added to this dict, + # so it can be found when later paths being restored are linked to + # it. + _restore_index_path = {} + def get_inode_key(cls, rorp): """Return rorp's key for _inode_ dictionaries""" return (rorp.getinode(), rorp.getdevloc()) @@ -101,26 +106,29 @@ class Hardlink: """True if rorp's index is already linked to something on src side""" return len(cls.get_indicies(rorp, 1)) >= 2 - def restore_link(cls, mirror_rel_index, rpath): + def restore_link(cls, index, rpath): """Restores a linked file by linking it When restoring, all the hardlink data is already present, and - we can only link to something already written. Returns true - if succeeded in creating rpath, false if must restore rpath - normally. + we can only link to something already written. In either + case, add to the _restore_index_path dict, so we know later + that the file is available for hard + linking. + + Returns true if succeeded in creating rpath, false if must + restore rpath normally. """ - full_index = mirror_rel_index + rpath.index - if not cls._src_index_indicies.has_key(full_index): return None - truncated_list = [] - for index in cls._src_index_indicies[full_index]: - if index[:len(mirror_rel_index)] == mirror_rel_index: - truncated_list.append(index[len(mirror_rel_index):]) - - if not truncated_list or truncated_list[0] >= rpath.index: return None - srclink = RPath(rpath.conn, rpath.base, truncated_list[0]) - rpath.hardlink(srclink.path) - return 1 + if index not in cls._src_index_indicies: return None + for linked_index in cls._src_index_indicies[index]: + if linked_index in cls._restore_index_path: + srcpath = cls._restore_index_path[linked_index] + Log("Restoring %s by hard linking to %s" % + (rpath.path, srcpath), 6) + rpath.hardlink(srcpath) + return 1 + cls._restore_index_path[index] = rpath.path + return None def link_rp(cls, src_rorp, dest_rpath, dest_root = None): """Make dest_rpath into a link analogous to that of src_rorp""" diff --git a/rdiff-backup/src/highlevel.py b/rdiff-backup/src/highlevel.py index d0bc743..36ba55a 100644 --- a/rdiff-backup/src/highlevel.py +++ b/rdiff-backup/src/highlevel.py @@ -24,12 +24,14 @@ class HighLevel: accompanying diagram. """ - def Mirror(src_rpath, dest_rpath, checkpoint = 1, session_info = None): + def Mirror(src_rpath, dest_rpath, checkpoint = 1, + session_info = None, write_finaldata = 1): """Turn dest_rpath into a copy of src_rpath Checkpoint true means to checkpoint periodically, otherwise not. If session_info is given, try to resume Mirroring from - that point. + that point. If write_finaldata is true, save extra data files + like hardlink_data. If it is false, make a complete mirror. """ SourceS = src_rpath.conn.HLSourceStruct @@ -40,7 +42,8 @@ class HighLevel: src_init_dsiter = SourceS.split_initial_dsiter() dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter) diffiter = SourceS.get_diffs_and_finalize(dest_sigiter) - DestS.patch_and_finalize(dest_rpath, diffiter, checkpoint) + DestS.patch_and_finalize(dest_rpath, diffiter, + checkpoint, write_finaldata) dest_rpath.setdata() @@ -61,24 +64,6 @@ class HighLevel: dest_rpath.setdata() inc_rpath.setdata() - def Restore(rest_time, mirror_base, rel_index, baseinc_tup, target_base): - """Like Restore.RestoreRecursive but check arguments""" - if (Globals.preserve_hardlinks != 0 and - Hardlink.retrieve_final(rest_time)): - Log("Hard link information found, attempting to preserve " - "hard links.", 4) - SetConnections.UpdateGlobal('preserve_hardlinks', 1) - else: SetConnections.UpdateGlobal('preserve_hardlinks', None) - - if not isinstance(target_base, DSRPath): - target_base = DSRPath(target_base.conn, target_base.base, - target_base.index, target_base.data) - if not isinstance(mirror_base, DSRPath): - mirror_base = DSRPath(mirror_base.conn, mirror_base.base, - mirror_base.index, mirror_base.data) - Restore.RestoreRecursive(rest_time, mirror_base, rel_index, - baseinc_tup, target_base) - MakeStatic(HighLevel) @@ -164,7 +149,7 @@ class HLDestinationStruct: def compare(src_rorp, dest_dsrp): """Return dest_dsrp if they are different, None if the same""" if not dest_dsrp: - dest_dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index) + dest_dsrp = cls.get_dsrp(baserp, src_rorp.index) if dest_dsrp.lstat(): Log("Warning: Found unexpected destination file %s, " "not processing it." % dest_dsrp.path, 2) @@ -203,8 +188,9 @@ class HLDestinationStruct: def get_dsrp(cls, dest_rpath, index): """Return initialized dsrp based on dest_rpath with given index""" - return DSRPath(source = None, dest_rpath.conn, - dest_rpath.base, index) + dsrp = DSRPath(None, dest_rpath.conn, dest_rpath.base, index) + if Globals.quoting_enabled: dsrp.quote_path() + return dsrp def get_finalizer(cls): """Return finalizer, starting from session info if necessary""" @@ -216,9 +202,13 @@ class HLDestinationStruct: """Return ITR, starting from state if necessary""" if cls._session_info and cls._session_info.ITR: return cls._session_info.ITR - else: return IncrementITR(inc_rpath) + else: + iitr = IncrementITR(inc_rpath) + iitr.override_changed() + return iitr - def patch_and_finalize(cls, dest_rpath, diffs, checkpoint = 1): + def patch_and_finalize(cls, dest_rpath, diffs, + checkpoint = 1, write_finaldata = 1): """Apply diffs and finalize""" collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) finalizer = cls.get_finalizer() @@ -242,7 +232,7 @@ class HLDestinationStruct: if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp) except: cls.handle_last_error(dsrp, finalizer) finalizer.Finish() - if Globals.preserve_hardlinks and Globals.rbdir: + if Globals.preserve_hardlinks and write_finaldata: Hardlink.final_writedata() if checkpoint: SaveState.checkpoint_remove() @@ -300,8 +290,7 @@ class HLDestinationStruct: Log.exception(1) if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1) else: SaveState.checkpoint_mirror(finalizer, dsrp, 1) - if Globals.preserve_hardlinks: - Hardlink.final_checkpoint(Globals.rbdir) + if Globals.preserve_hardlinks: Hardlink.final_checkpoint(Globals.rbdir) SaveState.touch_last_file_definitive() raise diff --git a/rdiff-backup/src/increment.py b/rdiff-backup/src/increment.py index 446806b..b28b315 100644 --- a/rdiff-backup/src/increment.py +++ b/rdiff-backup/src/increment.py @@ -1,4 +1,4 @@ -execfile("selection.py") +execfile("filename_mapping.py") ####################################################################### # @@ -85,10 +85,12 @@ class Inc: """Get new increment rp with given time suffix""" addtostr = lambda s: "%s.%s.%s" % (s, timestr, typestr) if rp.index: - return rp.__class__(rp.conn, rp.base, rp.index[:-1] + - (addtostr(rp.index[-1]),)) - else: return rp.__class__(rp.conn, addtostr(rp.base), rp.index) - + incrp = rp.__class__(rp.conn, rp.base, rp.index[:-1] + + (addtostr(rp.index[-1]),)) + else: incrp = rp.__class__(rp.conn, addtostr(rp.base), rp.index) + if Globals.quoting_enabled: incrp.quote_path() + return incrp + inctime = 0 while 1: inctime = Resume.FindTime(rp.index, inctime) @@ -123,7 +125,7 @@ class IncrementITR(IterTreeReducer): def __init__(self, inc_rpath): """Set inc_rpath, an rpath of the base of the tree""" self.inc_rpath = inc_rpath - IterTreeReducer.__init__(inc_rpath) + IterTreeReducer.__init__(self, inc_rpath) def start_process(self, index, diff_rorp, dsrp): """Initial processing of file @@ -133,11 +135,21 @@ class IncrementITR(IterTreeReducer): """ incpref = self.inc_rpath.new_index(index) + if Globals.quoting_enabled: incpref.quote_path() if dsrp.isdir(): self.init_dir(dsrp, diff_rorp, incpref) self.setvals(diff_rorp, dsrp, incpref) else: self.init_non_dir(dsrp, diff_rorp, incpref) + def override_changed(self): + """Set changed flag to true + + This is used only at the top level of a backup, to make sure + that a marker is created recording every backup session. + + """ + self.changed = 1 + def setvals(self, diff_rorp, dsrp, incpref): """Record given values in state dict since in directory @@ -162,7 +174,7 @@ class IncrementITR(IterTreeReducer): """ if not (incpref.lstat() and incpref.isdir()): incpref.mkdir() if diff_rorp and diff_rorp.isreg() and diff_rorp.file: - tf = TempFileManager(dsrp) + tf = TempFileManager.new(dsrp) RPathStatic.copy_with_attribs(diff_rorp, tf) tf.set_attached_filetype(diff_rorp.get_attached_filetype()) self.directory_replacement = tf @@ -170,7 +182,7 @@ class IncrementITR(IterTreeReducer): def init_non_dir(self, dsrp, diff_rorp, incpref): """Process a non directory file (initial pass)""" if not diff_rorp: return # no diff, so no change necessary - if diff_rorp.isreg and (dsrp.isreg() or diff_rorp.isflaglinked()): + if diff_rorp.isreg() and (dsrp.isreg() or diff_rorp.isflaglinked()): tf = TempFileManager.new(dsrp) def init_thunk(): if diff_rorp.isflaglinked(): @@ -180,8 +192,8 @@ class IncrementITR(IterTreeReducer): Inc.Increment_action(tf, dsrp, incpref).execute() Robust.make_tf_robustaction(init_thunk, (tf,), (dsrp,)).execute() else: - Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incref), - RORPIter.patchonce_action(none, dsrp, diff_rorp)] + Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incpref), + RORPIter.patchonce_action(None, dsrp, diff_rorp)] ).execute() self.changed = 1 @@ -207,89 +219,3 @@ class IncrementITR(IterTreeReducer): - - - def make_patch_increment_ITR(inc_rpath, initial_state = None): - """Return IterTreeReducer that patches and increments""" - def base_init(indexed_tuple): - """Patch if appropriate, return (a,b) tuple - - a is true if found directory and thus didn't take action - - if a is false, b is true if some changes were made - - if a is true, b is the rp of a temporary file used to hold - the diff_rorp's data (for dir -> normal file change), and - false if none was necessary. - - """ - diff_rorp, dsrp = indexed_tuple - incpref = inc_rpath.new_index(indexed_tuple.index) - if dsrp.isdir(): return init_dir(dsrp, diff_rorp, incpref) - else: return init_non_dir(dsrp, diff_rorp, incpref) - - def init_dir(dsrp, diff_rorp, incpref): - """Initial processing of a directory - - Make the corresponding directory right away, but wait - until the end to write the replacement. However, if the - diff_rorp contains data, we must write it locally before - continuing, or else that data will be lost in the stream. - - """ - if not (incpref.lstat() and incpref.isdir()): incpref.mkdir() - if diff_rorp and diff_rorp.isreg() and diff_rorp.file: - tf = TempFileManager.new(dsrp) - RPathStatic.copy_with_attribs(diff_rorp, tf) - tf.set_attached_filetype(diff_rorp.get_attached_filetype()) - return (1, tf) - else: return (1, None) - - def init_non_dir(dsrp, diff_rorp, incpref): - """Initial processing of non-directory - - If a reverse diff is called for it is generated by apply - the forwards diff first on a temporary file. - - """ - if diff_rorp: - if diff_rorp.isreg() and (dsrp.isreg() or - diff_rorp.isflaglinked()): - tf = TempFileManager.new(dsrp) - def init_thunk(): - if diff_rorp.isflaglinked(): - Hardlink.link_rp(diff_rorp, tf, dsrp) - else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp, - tf).execute() - Inc.Increment_action(tf, dsrp, incpref).execute() - Robust.make_tf_robustaction(init_thunk, (tf,), - (dsrp,)).execute() - else: - Robust.chain([Inc.Increment_action(diff_rorp, dsrp, - incpref), - RORPIter.patchonce_action( - None, dsrp, diff_rorp)]).execute() - return (None, 1) - return (None, None) - - def base_final(base_tuple, base_init_tuple, changed): - """Patch directory if not done, return true iff made change""" - if base_init_tuple[0]: # was directory - diff_rorp, dsrp = base_tuple - if changed or diff_rorp: - if base_init_tuple[1]: diff_rorp = base_init_tuple[1] - Inc.Increment(diff_rorp, dsrp, - inc_rpath.new_index(base_tuple.index)) - if diff_rorp: - RORPIter.patchonce_action(None, dsrp, - diff_rorp).execute() - if isinstance(diff_rorp, TempFile): diff_rorp.delete() - return 1 - return None - else: # changed iff base_init_tuple says it was - return base_init_tuple[1] - - return IterTreeReducer(base_init, lambda x,y: x or y, None, - base_final, initial_state) - - diff --git a/rdiff-backup/src/lazy.py b/rdiff-backup/src/lazy.py index 80cfa95..1bb2e2c 100644 --- a/rdiff-backup/src/lazy.py +++ b/rdiff-backup/src/lazy.py @@ -260,7 +260,7 @@ class IterTreeReducer: """ index = args[0] - assert type(index) is types.TupleType + assert type(index) is types.TupleType, type(index) if self.index is None: self.start_process(*args) diff --git a/rdiff-backup/src/main.py b/rdiff-backup/src/main.py index 557cefd..162cecf 100755 --- a/rdiff-backup/src/main.py +++ b/rdiff-backup/src/main.py @@ -22,20 +22,21 @@ class Main: try: return open(filename, "r") except IOError: Log.FatalError("Error opening file %s" % filename) - try: optlist, self.args = getopt.getopt(sys.argv[1:], "blmsv:V", + try: optlist, self.args = getopt.getopt(sys.argv[1:], "blmr:sv:V", ["backup-mode", "change-source-perms", - "checkpoint-interval=", "current-time=", "exclude=", - "exclude-device-files", "exclude-filelist=", - "exclude-filelist-stdin", "exclude-mirror=", - "exclude-regexp=", "force", "include=", - "include-filelist=", "include-filelist-stdin", - "include-regexp=", "list-increments", "mirror-only", - "no-compression", "no-compression-regexp=", - "no-hard-links", "no-resume", "remote-cmd=", - "remote-schema=", "remove-older-than=", "resume", - "resume-window=", "server", "terminal-verbosity=", - "test-server", "verbosity", "version", - "windows-time-format"]) + "chars-to-quote=", "checkpoint-interval=", + "current-time=", "exclude=", "exclude-device-files", + "exclude-filelist=", "exclude-filelist-stdin", + "exclude-mirror=", "exclude-regexp=", "force", + "include=", "include-filelist=", + "include-filelist-stdin", "include-regexp=", + "list-increments", "mirror-only", "no-compression", + "no-compression-regexp=", "no-hard-links", "no-resume", + "parsable-output", "quoting-char=", "remote-cmd=", + "remote-schema=", "remove-older-than=", + "restore-as-of=", "resume", "resume-window=", "server", + "terminal-verbosity=", "test-server", "verbosity", + "version", "windows-mode", "windows-time-format"]) except getopt.error: self.commandline_error("Error parsing commandline options") @@ -43,6 +44,9 @@ class Main: if opt == "-b" or opt == "--backup-mode": self.action = "backup" elif opt == "--change-source-perms": Globals.set('change_source_perms', 1) + elif opt == "--chars-to-quote": + Globals.set('chars_to_quote', arg) + Globals.set('quoting_enabled', 1) elif opt == "--checkpoint-interval": Globals.set_integer('checkpoint_interval', arg) elif opt == "--current-time": @@ -75,6 +79,13 @@ class Main: Globals.set("no_compression_regexp_string", arg) elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0) elif opt == '--no-resume': Globals.resume = 0 + elif opt == "-r" or opt == "--restore-as-of": + self.restore_timestr = arg + self.action = "restore-as-of" + elif opt == "--parsable-output": Globals.set('parsable_output', 1) + elif opt == "--quoting-char": + Globals.set('quoting_char', arg) + Globals.set('quoting_enabled', 1) elif opt == "--remote-cmd": self.remote_cmd = arg elif opt == "--remote-schema": self.remote_schema = arg elif opt == "--remove-older-than": @@ -84,14 +95,16 @@ class Main: elif opt == '--resume-window': Globals.set_integer('resume_window', arg) elif opt == "-s" or opt == "--server": self.action = "server" - elif opt == "--terminal-verbosity": - Log.setterm_verbosity(arg) + elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg) elif opt == "--test-server": self.action = "test-server" elif opt == "-V" or opt == "--version": print "rdiff-backup " + Globals.version sys.exit(0) - elif opt == "-v" or opt == "--verbosity": - Log.setverbosity(arg) + elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg) + elif opt == "--windows-mode": + Globals.set('time_separator', "_") + Globals.set('chars_to_quote', ":") + Globals.set('quoting_enabled', 1) elif opt == '--windows-time-format': Globals.set('time_separator', "_") else: Log.FatalError("Unknown option %s" % opt) @@ -112,7 +125,8 @@ class Main: self.commandline_error("No arguments given") if l > 0 and self.action == "server": self.commandline_error("Too many arguments given") - if l < 2 and (self.action == "backup" or self.action == "mirror"): + if l < 2 and (self.action == "backup" or self.action == "mirror" or + self.action == "restore-as-of"): self.commandline_error("Two arguments are required " "(source, destination).") if l == 2 and (self.action == "list-increments" or @@ -136,6 +150,8 @@ class Main: for rp in rps: rp.setdata() # Update with userinfo os.umask(077) + Time.setcurtime(Globals.current_time) + FilenameMapping.set_init_quote_vals() # This is because I originally didn't think compiled regexps # could be pickled, and so must be compiled on remote side. @@ -147,7 +163,8 @@ class Main: if self.action == "server": PipeConnection(sys.stdin, sys.stdout).Server() elif self.action == "backup": self.Backup(rps[0], rps[1]) - elif self.action == "restore": apply(self.Restore, rps) + elif self.action == "restore": self.Restore(*rps) + elif self.action == "restore-as-of": self.RestoreAsOf(rps[0], rps[1]) elif self.action == "mirror": self.Mirror(rps[0], rps[1]) elif self.action == "test-server": SetConnections.TestConnections() elif self.action == "list-increments": self.ListIncrements(rps[0]) @@ -175,7 +192,12 @@ class Main: """Turn dest_path into a copy of src_path""" Log("Mirroring %s to %s" % (src_rp.path, dest_rp.path), 5) self.mirror_check_paths(src_rp, dest_rp) - HighLevel.Mirror(src_rp, dest_rp, None) # No checkpointing - no rbdir + # Since no "rdiff-backup-data" dir, use root of destination. + SetConnections.UpdateGlobal('rbdir', dest_rp) + SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn) + RSI = Globals.backup_writer.Resume.ResumeCheck() + SaveState.init_filenames(None) + HighLevel.Mirror(src_rp, dest_rp, 1, RSI, None) def mirror_check_paths(self, rpin, rpout): """Check paths and return rpin, rpout""" @@ -193,7 +215,6 @@ rdiff-backup with the --force option if you want to mirror anyway.""" % SetConnections.BackupInitConnections(rpin.conn, rpout.conn) self.backup_init_select(rpin, rpout) self.backup_init_dirs(rpin, rpout) - Time.setcurtime(Globals.current_time) RSI = Globals.backup_writer.Resume.ResumeCheck() if self.prevtime: Time.setprevtime(self.prevtime) @@ -206,8 +227,9 @@ rdiff-backup with the --force option if you want to mirror anyway.""" % def backup_init_select(self, rpin, rpout): """Create Select objects on source and dest connections""" - rpin.conn.Globals.set_select(1, rpin, self.select_opts) - rpout.conn.Globals.set_select(None, rpout, self.select_mirror_opts) + rpin.conn.Globals.set_select(DSRPath(1, rpin), self.select_opts) + rpout.conn.Globals.set_select(DSRPath(None, rpout), + self.select_mirror_opts, 1) def backup_init_dirs(self, rpin, rpout): """Make sure rpin and rpout are valid, init data dir and logging""" @@ -267,9 +289,8 @@ may need to use the --exclude option.""" % (rpout.path, rpin.path), 2) def backup_get_mirrorrps(self): """Return list of current_mirror rps""" if not self.datadir.isdir(): return [] - mirrorfiles = filter(lambda f: f.startswith("current_mirror."), - self.datadir.listdir()) - mirrorrps = map(lambda x: self.datadir.append(x), mirrorfiles) + mirrorrps = [self.datadir.append(fn) for fn in self.datadir.listdir() + if fn.startswith("current_mirror.")] return filter(lambda rp: rp.isincfile(), mirrorrps) def backup_get_mirrortime(self): @@ -299,22 +320,45 @@ went wrong during your last backup? Using """ + mirrorrps[-1].path, 2) def Restore(self, src_rp, dest_rp = None): - """Main restoring function - take src_path to dest_path""" - Log("Starting Restore", 5) + """Main restoring function + + Here src_rp should be an increment file, and if dest_rp is + missing it defaults to the base of the increment. + + """ rpin, rpout = self.restore_check_paths(src_rp, dest_rp) - self.restore_init_select(rpin, rpout) - inc_tup = self.restore_get_inctup(rpin) - mirror_base, mirror_rel_index = self.restore_get_mirror(rpin) - rtime = Time.stringtotime(rpin.getinctime()) + time = Time.stringtotime(rpin.getinctime()) + self.restore_common(rpin, rpout, time) + + def RestoreAsOf(self, rpin, target): + """Secondary syntax for restore operation + + rpin - RPath of mirror file to restore (not nec. with correct index) + target - RPath of place to put restored file + + """ + self.restore_check_paths(rpin, target, 1) + try: time = Time.genstrtotime(self.restore_timestr) + except TimeError, exp: Log.FatalError(str(exp)) + self.restore_common(rpin, target, time) + + def restore_common(self, rpin, target, time): + """Restore operation common to Restore and RestoreAsOf""" + Log("Starting Restore", 5) + mirror_root, index = self.restore_get_root(rpin) + mirror = mirror_root.new_index(index) + inc_rpath = self.datadir.append_path('increments', index) + self.restore_init_select(mirror_root, target) Log.open_logfile(self.datadir.append("restore.log")) - HighLevel.Restore(rtime, mirror_base, mirror_rel_index, inc_tup, rpout) + Restore.Restore(inc_rpath, mirror, target, time) - def restore_check_paths(self, rpin, rpout): + def restore_check_paths(self, rpin, rpout, restoreasof = None): """Check paths and return pair of corresponding rps""" - if not rpin.lstat(): - Log.FatalError("Increment file %s does not exist" % rpin.path) - if not rpin.isincfile(): - Log.FatalError("""File %s does not look like an increment file. + if not restoreasof: + if not rpin.lstat(): + Log.FatalError("Source file %s does not exist" % rpin.path) + elif not rpin.isincfile(): + Log.FatalError("""File %s does not look like an increment file. Try restoring from an increment file (the filenames look like "foobar.2001-09-01T04:49:04-07:00.diff").""" % rpin.path) @@ -322,8 +366,8 @@ Try restoring from an increment file (the filenames look like if not rpout: rpout = RPath(Globals.local_connection, rpin.getincbase_str()) if rpout.lstat(): - Log.FatalError("Restore target %s already exists. " - "Will not overwrite." % rpout.path) + Log.FatalError("Restore target %s already exists," + "and will not be overwritten." % rpout.path) return rpin, rpout def restore_init_select(self, rpin, rpout): @@ -334,81 +378,64 @@ Try restoring from an increment file (the filenames look like the restore operation isn't. """ - Globals.set_select(1, rpin, self.select_mirror_opts) - Globals.set_select(None, rpout, self.select_opts) - - def restore_get_inctup(self, rpin): - """Return increment tuple (incrp, list of incs)""" - rpin_dir = rpin.dirsplit()[0] - if not rpin_dir: rpin_dir = "/" - rpin_dir_rp = RPath(rpin.conn, rpin_dir) - incbase = rpin.getincbase() - incbasename = incbase.dirsplit()[1] - inclist = filter(lambda rp: rp.isincfile() and - rp.getincbase_str() == incbasename, - map(rpin_dir_rp.append, rpin_dir_rp.listdir())) - return IndexedTuple((), (incbase, inclist)) - - def restore_get_mirror(self, rpin): - """Return (mirror file, relative index) and set the data dir + Globals.set_select(DSRPath(1, rpin), self.select_mirror_opts) + Globals.set_select(DSRPath(None, rpout), self.select_opts) + + def restore_get_root(self, rpin): + """Return (mirror root, index) and set the data dir The idea here is to keep backing up on the path until we find - something named "rdiff-backup-data". Then use that as a - reference to calculate the oldfile. This could fail if the - increment file is pointed to in a funny way, using symlinks or - somesuch. + a directory that contains "rdiff-backup-data". That is the + mirror root. If the path from there starts + "rdiff-backup-data/increments*", then the index is the + remainder minus that. Otherwise the index is just the path + minus the root. - The mirror file will have index (), so also return the index - relative to the rootrp. + All this could fail if the increment file is pointed to in a + funny way, using symlinks or somesuch. """ - pathcomps = os.path.join(rpin.conn.os.getcwd(), - rpin.getincbase().path).split("/") - for i in range(1, len(pathcomps)): - datadirrp = RPath(rpin.conn, "/".join(pathcomps[:i+1])) - if pathcomps[i] == "rdiff-backup-data" and datadirrp.isdir(): - break - else: Log.FatalError("Unable to find rdiff-backup-data dir") - - Globals.rbdir = self.datadir = datadirrp - rootrp = RPath(rpin.conn, "/".join(pathcomps[:i])) - if not rootrp.lstat(): - Log.FatalError("Root of mirror area %s does not exist" % - rootrp.path) - else: Log("Using root mirror %s" % rootrp.path, 6) - - from_datadir = pathcomps[i+1:] - if not from_datadir: raise RestoreError("Problem finding mirror file") - rel_index = tuple(from_datadir[1:]) - mirrorrp = RPath(rootrp.conn, - apply(os.path.join, (rootrp.path,) + rel_index)) - Log("Using mirror file %s" % mirrorrp.path, 6) - return (mirrorrp, rel_index) - - - def ListIncrements(self, rootrp): - """Print out a summary of the increments and their times""" - datadir = self.li_getdatadir(rootrp, - """Unable to open rdiff-backup-data dir. - -The argument to rdiff-backup -l or rdiff-backup --list-increments -should be the root of the target backup directory, of which -rdiff-backup-data is a subdirectory. So, if you ran - -rdiff-backup /home/foo /mnt/back/bar + if rpin.isincfile(): relpath = rpin.getincbase().path + else: relpath = rpin.path + pathcomps = os.path.join(rpin.conn.os.getcwd(), relpath).split("/") + assert len(pathcomps) >= 2 # path should be relative to / + + i = len(pathcomps) + while i >= 2: + parent_dir = RPath(rpin.conn, "/".join(pathcomps[:i])) + if (parent_dir.isdir() and + "rdiff-backup-data" in parent_dir.listdir()): break + i = i-1 + else: Log.FatalError("Unable to find rdiff-backup-data directory") + + self.rootrp = rootrp = parent_dir + Log("Using mirror root directory %s" % rootrp.path, 6) + + self.datadir = rootrp.append_path("rdiff-backup-data") + SetConnections.UpdateGlobal('rbdir', self.datadir) + if not self.datadir.isdir(): + Log.FatalError("Unable to read rdiff-backup-data directory %s" % + self.datadir.path) -earlier, try: + from_datadir = tuple(pathcomps[i:]) + if not from_datadir or from_datadir[0] != "rdiff-backup-data": + return (rootrp, from_datadir) # in mirror, not increments + assert from_datadir[1] == "increments" + return (rootrp, from_datadir[2:]) -rdiff-backup -l /mnt/back/bar -""") - print Manage.describe_root_incs(datadir) - def li_getdatadir(self, rootrp, errormsg): - """Return data dir if can find it, otherwise use errormsg""" - datadir = rootrp.append("rdiff-backup-data") - if not datadir.lstat() or not datadir.isdir(): - Log.FatalError(errormsg) - return datadir + def ListIncrements(self, rp): + """Print out a summary of the increments and their times""" + mirror_root, index = self.restore_get_root(rp) + Globals.rbdir = datadir = \ + mirror_root.append_path("rdiff-backup-data") + mirrorrp = mirror_root.new_index(index) + inc_rpath = datadir.append_path('increments', index) + incs = Restore.get_inclist(inc_rpath) + mirror_time = Restore.get_mirror_time() + if Globals.parsable_output: + print Manage.describe_incs_parsable(incs, mirror_time, mirrorrp) + else: print Manage.describe_incs_human(incs, mirror_time, mirrorrp) def RemoveOlderThan(self, rootrp): @@ -417,7 +444,8 @@ rdiff-backup -l /mnt/back/bar """Unable to open rdiff-backup-data dir. Try finding the increments first using --list-increments.""") - time = self.rot_get_earliest_time() + try: time = Time.genstrtotime(self.remove_older_than_string) + except TimeError, exp: Log.FatalError(str(exp)) timep = Time.timetopretty(time) Log("Deleting increment(s) before %s" % timep, 4) incobjs = filter(lambda x: x.time < time, Manage.get_incobjs(datadir)) @@ -433,11 +461,6 @@ Try finding the increments first using --list-increments.""") incobjs_time), 3) Manage.delete_earlier_than(datadir, time) - def rot_get_earliest_time(self): - """Return earliest time in seconds that will not be deleted""" - seconds = Time.intstringtoseconds(self.remove_older_than_string) - return time.time() - seconds - if __name__ == "__main__" and not globals().has_key('__no_execute__'): diff --git a/rdiff-backup/src/manage.py b/rdiff-backup/src/manage.py index c0f4a85..0c08872 100644 --- a/rdiff-backup/src/manage.py +++ b/rdiff-backup/src/manage.py @@ -12,37 +12,53 @@ class Manage: """Return Increments objects given the rdiff-backup data directory""" return map(IncObj, Manage.find_incrps_with_base(datadir, "increments")) - def find_incrps_with_base(dir_rp, basename): - """Return list of incfiles with given basename in dir_rp""" - rps = map(dir_rp.append, dir_rp.listdir()) - incrps = filter(RPath.isincfile, rps) - result = filter(lambda rp: rp.getincbase_str() == basename, incrps) - Log("find_incrps_with_base: found %d incs" % len(result), 6) - return result + def get_file_type(rp): + """Returns one of "regular", "directory", "missing", or "special".""" + if not rp.lstat(): return "missing" + elif rp.isdir(): return "directory" + elif rp.isreg(): return "regular" + else: return "special" - def describe_root_incs(datadir): + def get_inc_type(inc): + """Return file type increment represents""" + assert inc.isincfile() + type = inc.getinctype() + if type == "dir": return "directory" + elif type == "diff": return "regular" + elif type == "missing": return "missing" + elif type == "snapshot": return Manage.get_file_type(inc) + else: assert None, "Unknown type %s" % (type,) + + def describe_incs_parsable(incs, mirror_time, mirrorrp): + """Return a string parsable by computer describing the increments + + Each line is a time in seconds of the increment, and then the + type of the file. It will be sorted oldest to newest. For example: + + 10000 regular + 20000 directory + 30000 special + 40000 missing + 50000 regular <- last will be the current mirror + + """ + incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs] + incpairs.sort() + result = ["%s %s" % (time, Manage.get_inc_type(inc)) + for time, inc in incpairs] + result.append("%s %s" % (mirror_time, Manage.get_file_type(mirrorrp))) + return "\n".join(result) + + def describe_incs_human(incs, mirror_time, mirrorrp): """Return a string describing all the the root increments""" - result = [] - currentrps = Manage.find_incrps_with_base(datadir, "current_mirror") - if not currentrps: - Log("Warning: no current mirror marker found", 1) - elif len(currentrps) > 1: - Log("Warning: multiple mirror markers found", 1) - for rp in currentrps: - result.append("Found mirror marker %s" % rp.path) - result.append("Indicating latest mirror taken at %s" % - Time.stringtopretty(rp.getinctime())) - result.append("---------------------------------------------" - "-------------") - - # Sort so they are in reverse order by time - time_w_incobjs = map(lambda io: (-io.time, io), - Manage.get_incobjs(datadir)) - time_w_incobjs.sort() - incobjs = map(lambda x: x[1], time_w_incobjs) - result.append("Found %d increments:" % len(incobjs)) - result.append("\n------------------------------------------\n".join( - map(IncObj.full_description, incobjs))) + incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs] + incpairs.sort() + + result = ["Found %d increments:" % len(incpairs)] + for time, inc in incpairs: + result.append(" %s %s" % + (inc.dirsplit()[1], Time.timetopretty(time))) + result.append("Current mirror: %s" % Time.timetopretty(mirror_time)) return "\n".join(result) def delete_earlier_than(baserp, time): @@ -53,6 +69,11 @@ class Manage: rdiff-backup-data directory should be the root of the tree. """ + baserp.conn.Manage.delete_earlier_than_local(baserp, time) + + def delete_earlier_than_local(baserp, time): + """Like delete_earlier_than, but run on local connection for speed""" + assert baserp.conn is Globals.local_connection def yield_files(rp): yield rp if rp.isdir(): diff --git a/rdiff-backup/src/restore.py b/rdiff-backup/src/restore.py index dcba7f3..0faa9b2 100644 --- a/rdiff-backup/src/restore.py +++ b/rdiff-backup/src/restore.py @@ -24,32 +24,78 @@ class Restore: same index as mirror. """ - if not isinstance(mirror, DSRPath): - mirror = DSRPath(source = 1, mirror) - if not isinstance(target, DSRPath): - target = DSRPath(source = None, target) + if not isinstance(mirror, DSRPath): mirror = DSRPath(1, mirror) + if not isinstance(target, DSRPath): target = DSRPath(None, target) + + mirror_time = Restore.get_mirror_time() + rest_time = Restore.get_rest_time(rest_time) + inc_list = Restore.get_inclist(inc_rpath) + rid = RestoreIncrementData(inc_rpath.index, inc_rpath, inc_list) + rid.sortincseq(rest_time, mirror_time) + Restore.check_hardlinks(rest_time) + Restore.restore_recursive(inc_rpath.index, mirror, rid, target, + rest_time, mirror_time) + + def get_mirror_time(): + """Return the time (in seconds) of latest mirror""" + current_mirror_incs = \ + Restore.get_inclist(Globals.rbdir.append("current_mirror")) + if not current_mirror_incs: + Log.FatalError("Could not get time of current mirror") + elif len(current_mirror_incs) > 1: + Log("Warning, two different dates for current mirror found", 2) + return Time.stringtotime(current_mirror_incs[0].getinctime()) + + def get_rest_time(old_rest_time): + """If old_rest_time is between two increments, return older time + + There is a slightly tricky reason for doing this: The rest of + the code just ignores increments that are older than + rest_time. But sometimes we want to consider the very next + increment older than rest time, because rest_time will be + between two increments, and what was actually on the mirror + side will correspond to the older one. + + So here we assume all rdiff-backup events were recorded in + "increments" increments, and if its in-between we pick the + older one here. + """ + base_incs = Restore.get_inclist(Globals.rbdir.append("increments")) + if not base_incs: return old_rest_time + inctimes = [Time.stringtotime(inc.getinctime()) for inc in base_incs] + return max(filter(lambda time: time <= old_rest_time, inctimes)) + + def get_inclist(inc_rpath): + """Returns increments with given base""" dirname, basename = inc_rpath.dirsplit() parent_dir = RPath(inc_rpath.conn, dirname, ()) index = inc_rpath.index - if inc_rpath.index: + if index: get_inc_ext = lambda filename: \ RPath(inc_rpath.conn, inc_rpath.base, inc_rpath.index[:-1] + (filename,)) else: get_inc_ext = lambda filename: \ - RPath(inc_rpath.conn, os.join(dirname, filename)) + RPath(inc_rpath.conn, os.path.join(dirname, filename)) inc_list = [] for filename in parent_dir.listdir(): inc = get_inc_ext(filename) - if inc.getincbase_str() == basename: inc_list.append(inc) - - rid = RestoreIncrementData(index, inc_rpath, inc_list) - rid.sortincseq(rest_time) - Restore.restore_recursive(index, mirror, rid, target, rest_time) - - def restore_recursive(index, mirror, rid, target, time): + if inc.isincfile() and inc.getincbase_str() == basename: + inc_list.append(inc) + return inc_list + + def check_hardlinks(rest_time): + """Check for hard links and enable hard link support if found""" + if (Globals.preserve_hardlinks != 0 and + Hardlink.retrieve_final(rest_time)): + Log("Hard link information found, attempting to preserve " + "hard links.", 5) + SetConnections.UpdateGlobal('preserve_hardlinks', 1) + else: SetConnections.UpdateGlobal('preserve_hardlinks', None) + + def restore_recursive(index, mirror, rid, target, time, mirror_time): """Recursive restore function. rid is a RestoreIncrementData object whose inclist is already @@ -66,14 +112,15 @@ class Restore: mirror_finalizer = DestructiveSteppingFinalizer() target_finalizer = DestructiveSteppingFinalizer() - for rcd in Restore.yield_rcds(rid.index, mirror, rid, target, time): + for rcd in Restore.yield_rcds(rid.index, mirror, rid, + target, time, mirror_time): rcd.RestoreFile() - if rcd.mirror: mirror_finalizer(rcd.mirror) - target_finalizer(rcd.target) + if rcd.mirror: mirror_finalizer(rcd.index, rcd.mirror) + target_finalizer(rcd.target.index, rcd.target) target_finalizer.Finish() mirror_finalizer.Finish() - def yield_rcds(index, mirrorrp, rid, target, rest_time): + def yield_rcds(index, mirrorrp, rid, target, rest_time, mirror_time): """Iterate RestoreCombinedData objects starting with given args rid is a RestoreCombinedData object. target is an rpath where @@ -91,9 +138,10 @@ class Restore: mirrorrp = None rcd = RestoreCombinedData(rid, mirrorrp, target) - if mirrorrp and mirrorrp.isdir() or rid and rid.inc_rpath.isdir(): + if mirrorrp and mirrorrp.isdir() or \ + rid and rid.inc_rpath and rid.inc_rpath.isdir(): sub_rcds = Restore.yield_sub_rcds(index, mirrorrp, rid, - target, rest_time) + target, rest_time, mirror_time) else: sub_rcds = None if select_result == 1: @@ -108,35 +156,39 @@ class Restore: yield first for sub_rcd in sub_rcds: yield sub_rcd - def yield_collated_tuples_dir(index, mirrorrp, rid, target, rest_time): + def yield_sub_rcds(index, mirrorrp, rid, target, rest_time, mirror_time): """Yield collated tuples from inside given args""" - if not Restore.check_dir_exists(mirrorrp, inc_tup): return + if not Restore.check_dir_exists(mirrorrp, rid): return mirror_iter = Restore.yield_mirrorrps(mirrorrp) - rid_iter = Restore.get_rids(rid, rest_time) + rid_iter = Restore.yield_rids(rid, rest_time, mirror_time) for indexed_tup in RORPIter.CollateIterators(mirror_iter, rid_iter): index = indexed_tup.index new_mirrorrp, new_rid = indexed_tup - for rcd in Restore.yield_collated_tuples(index, new_mirrorrp, - new_rid, target.new_index(index), rest_time): + for rcd in Restore.yield_rcds(index, new_mirrorrp, + new_rid, target.append(index[-1]), rest_time, mirror_time): yield rcd - def check_dir_exists(mirrorrp, inc_tuple): + def check_dir_exists(mirrorrp, rid): """Return true if target should be a directory""" - if inc_tuple and inc_tuple[1]: + if rid and rid.inc_list: # Incs say dir if last (earliest) one is a dir increment - return inc_tuple[1][-1].getinctype() == "dir" + return rid.inc_list[-1].getinctype() == "dir" elif mirrorrp: return mirrorrp.isdir() # if no incs, copy mirror else: return None def yield_mirrorrps(mirrorrp): """Yield mirrorrps underneath given mirrorrp""" if mirrorrp and mirrorrp.isdir(): - dirlist = mirrorrp.listdir() - dirlist.sort() - for filename in dirlist: yield mirrorrp.append(filename) - - def yield_rids(rid, rest_time): + if Globals.quoting_enabled: + for rp in FilenameMapping.get_quoted_dir_children(mirrorrp): + yield rp + else: + dirlist = mirrorrp.listdir() + dirlist.sort() + for filename in dirlist: yield mirrorrp.append(filename) + + def yield_rids(rid, rest_time, mirror_time): """Yield RestoreIncrementData objects within given rid dir If the rid doesn't correspond to a directory, don't yield any @@ -148,16 +200,19 @@ class Restore: if not rid or not rid.inc_rpath or not rid.inc_rpath.isdir(): return rid_dict = {} # dictionary of basenames:rids dirlist = rid.inc_rpath.listdir() + if Globals.quoting_enabled: + dirlist = [FilenameMapping.unquote(fn) for fn in dirlist] def affirm_dict_indexed(basename): """Make sure the rid dictionary has given basename as key""" - if not inc_list_dict.has_key(basename): + if not rid_dict.has_key(basename): rid_dict[basename] = RestoreIncrementData( rid.index + (basename,), None, []) # init with empty rid def add_to_dict(filename): """Add filename to the inc tuple dictionary""" rp = rid.inc_rpath.append(filename) + if Globals.quoting_enabled: rp.quote_path() if rp.isincfile(): basename = rp.getincbase_str() affirm_dict_indexed(basename) @@ -167,14 +222,14 @@ class Restore: rid_dict[filename].inc_rpath = rp for filename in dirlist: add_to_dict(filename) - keys = inc_list_dict.keys() + keys = rid_dict.keys() keys.sort() # sortincseq now to avoid descending .missing directories later for key in keys: rid = rid_dict[key] if rid.inc_rpath or rid.inc_list: - rid.sortincseq(rest_time) + rid.sortincseq(rest_time, mirror_time) yield rid MakeStatic(Restore) @@ -192,26 +247,36 @@ class RestoreIncrementData: self.inc_rpath = inc_rpath self.inc_list = inc_list - def sortincseq(self, rest_time): + def sortincseq(self, rest_time, mirror_time): """Sort self.inc_list sequence, throwing away irrelevant increments""" - incpairs = map(lambda rp: (Time.stringtotime(rp.getinctime()), rp), - self.inc_list) - # Only consider increments at or after the time being restored - incpairs = filter(lambda pair: pair[0] >= rest_time, incpairs) + if not self.inc_list or rest_time >= mirror_time: + self.inc_list = [] + return - # Now throw away older unnecessary increments - incpairs.sort() + newer_incs = self.get_newer_incs(rest_time, mirror_time) i = 0 - while(i < len(incpairs)): + while(i < len(newer_incs)): # Only diff type increments require later versions - if incpairs[i][1].getinctype() != "diff": break + if newer_incs[i].getinctype() != "diff": break i = i+1 - incpairs = incpairs[:i+1] + self.inc_list = newer_incs[:i+1] + self.inc_list.reverse() # return in reversed order (latest first) + + def get_newer_incs(self, rest_time, mirror_time): + """Return list of newer incs sorted by time (increasing) - # Return increments in reversed order (latest first) - incpairs.reverse() - self.inc_list = map(lambda pair: pair[1], incpairs) + Also discard increments older than rest_time (rest_time we are + assuming is the exact time rdiff-backup was run, so no need to + consider the next oldest increment or any of that) + """ + incpairs = [] + for inc in self.inc_list: + time = Time.stringtotime(inc.getinctime()) + if time >= rest_time: incpairs.append((time, inc)) + incpairs.sort() + return [pair[1] for pair in incpairs] + class RestoreCombinedData: """Combine index information from increment and mirror directories @@ -235,9 +300,12 @@ class RestoreCombinedData: if mirror: self.mirror = mirror assert mirror.index == self.index + else: self.mirror = None elif mirror: self.index = mirror.index self.mirror = mirror + self.inc_list = [] + self.inc_rpath = None else: assert None, "neither rid nor mirror given" self.target = target @@ -249,15 +317,15 @@ class RestoreCombinedData: if self.restore_hardlink(): return - if not inclist or inclist[0].getinctype() == "diff": + if not self.inc_list or self.inc_list[0].getinctype() == "diff": assert self.mirror and self.mirror.lstat(), \ "No base to go with incs for %s" % self.target.path RPath.copy_with_attribs(self.mirror, self.target) for inc in self.inc_list: self.applyinc(inc, self.target) - def log(self) + def log(self): """Log current restore action""" - inc_string = ','.join(map(lambda x: x.path, self.inc_list)) + inc_string = ','.join([inc.path for inc in self.inc_list]) Log("Restoring %s with increments %s to %s" % (self.mirror and self.mirror.path, inc_string, self.target.path), 5) @@ -266,7 +334,7 @@ class RestoreCombinedData: """Hard link target and return true if hard linking appropriate""" if (Globals.preserve_hardlinks and Hardlink.restore_link(self.index, self.target)): - RPath.copy_attribs(self.inc_list and inc_list[-1] or + RPath.copy_attribs(self.inc_list and self.inc_list[-1] or self.mirror, self.target) return 1 return None diff --git a/rdiff-backup/src/robust.py b/rdiff-backup/src/robust.py index 22f35b9..a71eabc 100644 --- a/rdiff-backup/src/robust.py +++ b/rdiff-backup/src/robust.py @@ -252,7 +252,7 @@ class TempFile(RPath): if self.isdir() and not rp_dest.isdir(): # Cannot move a directory directly over another file rp_dest.delete() - if (isinstance(rp_dest, DSRPath) and rp_dest.perms_delayed + if (isinstance(rp_dest, DSRPath) and rp_dest.delay_perms and not self.hasfullperms()): # If we are moving to a delayed perm directory, delay # permission change on destination. @@ -531,7 +531,7 @@ class Resume: Log("Last backup dated %s was aborted, but we aren't " "resuming it." % Time.timetopretty(si.time), 2) return None - assert 0 + assert None MakeClass(Resume) diff --git a/rdiff-backup/src/rpath.py b/rdiff-backup/src/rpath.py index 0089bf6..110b89f 100644 --- a/rdiff-backup/src/rpath.py +++ b/rdiff-backup/src/rpath.py @@ -168,6 +168,17 @@ class RPathStatic: rp_dest.data = rp_source.data rp_source.data = {'type': None} + # If we are moving to a DSRPath, assume that the current times + # are the intended ones. We need to save them now in case + # they are changed later. + if isinstance(rp_dest, DSRPath): + if rp_dest.delay_mtime: + if 'mtime' in rp_dest.data: + rp_dest.setmtime(rp_dest.data['mtime']) + if rp_dest.delay_atime: + if 'atime' in rp_dest.data: + rp_dest.setatime(rp_dest.data['atime']) + def tupled_lstat(filename): """Like os.lstat, but return only a tuple, or None if os.error @@ -413,7 +424,7 @@ class RPath(RORPath): self.base = base self.path = apply(os.path.join, (base,) + self.index) self.file = None - if data: self.data = data + if data or base is None: self.data = data else: self.setdata() def __str__(self): @@ -493,6 +504,12 @@ class RPath(RORPath): s = self.conn.reval("lambda path: os.lstat(path).st_rdev", self.path) return (s >> 8, s & 0xff) + def quote_path(self): + """Set path from quoted version of index""" + quoted_list = [FilenameMapping.quote(path) for path in self.index] + self.path = apply(os.path.join, [self.base] + quoted_list) + self.setdata() + def chmod(self, permissions): """Wrapper around os.chmod""" self.conn.os.chmod(self.path, permissions) @@ -594,7 +611,8 @@ class RPath(RORPath): if not self.lstat(): return # must have been deleted in meantime elif self.isdir(): itm = RpathDeleter() - for dsrp in Select(self, None).set_iter(): itm(dsrp.index, dsrp) + for dsrp in Select(DSRPath(None, self)).set_iter(): + itm(dsrp.index, dsrp) itm.Finish() else: self.conn.os.unlink(self.path) self.setdata() @@ -616,7 +634,7 @@ class RPath(RORPath): self.path.split("/"))) if self.path[0] == "/": newpath = "/" + newpath elif not newpath: newpath = "." - return self.__class__(self.conn, newpath, ()) + return self.newpath(newpath) def dirsplit(self): """Returns a tuple of strings (dirname, basename) @@ -635,10 +653,20 @@ class RPath(RORPath): comps = normed.path.split("/") return "/".join(comps[:-1]), comps[-1] + def newpath(self, newpath, index = ()): + """Return new RPath with the same connection but different path""" + return self.__class__(self.conn, newpath, index) + def append(self, ext): """Return new RPath with same connection by adjoing ext""" return self.__class__(self.conn, self.base, self.index + (ext,)) + def append_path(self, ext, new_index = ()): + """Like append, but add ext to path instead of to index""" + assert not self.index # doesn't make sense if index isn't () + return self.__class__(self.conn, os.path.join(self.base, ext), + new_index) + def new_index(self, index): """Return similar RPath but with new index""" return self.__class__(self.conn, self.base, index) diff --git a/rdiff-backup/src/selection.py b/rdiff-backup/src/selection.py index aaa8639..cae6db3 100644 --- a/rdiff-backup/src/selection.py +++ b/rdiff-backup/src/selection.py @@ -60,21 +60,20 @@ class Select: # This re should not match normal filenames, but usually just globs glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S) - def __init__(self, rpath, source): - """DSRPIterator initializer. + def __init__(self, dsrpath, quoted_filenames = None): + """DSRPIterator initializer. dsrp is the root directory - rpath is the root dir. Source is true if rpath is the root of - the source directory, and false for the mirror directory + When files have quoted characters in them, quoted_filenames + should be true. Then RPath's index will be the unquoted + version. """ - assert isinstance(rpath, RPath) + assert isinstance(dsrpath, DSRPath) self.selection_functions = [] - self.source = source - if isinstance(rpath, DSRPath): self.dsrpath = rpath - else: self.dsrpath = DSRPath(rpath.conn, rpath.base, - rpath.index, rpath.data) + self.dsrpath = dsrpath self.prefix = self.dsrpath.path - + self.quoting_on = Globals.quoting_enabled and quoted_filenames + def set_iter(self, starting_index = None, sel_func = None): """Initialize more variables, get ready to iterate @@ -106,7 +105,7 @@ class Select: """ s = sel_func(dsrpath) - if s === 0: return + if s == 0: return elif s == 1: # File is included yield dsrpath if dsrpath.isdir(): @@ -122,11 +121,15 @@ class Select: def iterate_in_dir(self, dsrpath, rec_func, sel_func): """Iterate the dsrps in directory dsrpath.""" - dir_listing = dsrpath.listdir() - dir_listing.sort() - for filename in dir_listing: - for dsrp in rec_func(dsrpath.append(filename), rec_func, sel_func): - yield dsrp + if self.quoting_on: + for subdir in FilenameMapping.get_quoted_dir_children(dsrpath): + for dsrp in rec_func(subdir, rec_func, sel_func): yield dsrp + else: + dir_listing = dsrpath.listdir() + dir_listing.sort() + for filename in dir_listing: + for dsrp in rec_func(dsrpath.append(filename), + rec_func, sel_func): yield dsrp def iterate_starting_from(self, dsrpath, rec_func, sel_func): """Like Iterate, but only yield indicies > self.starting_index""" diff --git a/rdiff-backup/src/ttime.py b/rdiff-backup/src/ttime.py index f6d041c..166c3f8 100644 --- a/rdiff-backup/src/ttime.py +++ b/rdiff-backup/src/ttime.py @@ -12,6 +12,7 @@ class Time: """Functions which act on the time""" _interval_conv_dict = {"s": 1, "m": 60, "h": 3600, "D": 86400, "W": 7*86400, "M": 30*86400, "Y": 365*86400} + _integer_regexp = re.compile("^[0-9]+$") _interval_regexp = re.compile("^([0-9]+)([smhDWMY])") _genstr_date_regexp1 = re.compile("^(?P[0-9]{4})[-/]" "(?P[0-9]{1,2})[-/](?P[0-9]{1,2})$") @@ -70,7 +71,7 @@ class Time: utc_in_secs = time.mktime(timetuple) - time.altzone else: utc_in_secs = time.mktime(timetuple) - time.timezone - return utc_in_secs + cls.tzdtoseconds(timestring[19:]) + return long(utc_in_secs) + cls.tzdtoseconds(timestring[19:]) except (TypeError, ValueError, AssertionError): return None def timetopretty(cls, timeinseconds): @@ -155,8 +156,10 @@ strings, like "2002-04-26T04:22:01-07:00" (strings like "2002-04-26T04:22:01" are also acceptable - rdiff-backup will use the current time zone), or ordinary dates like 2/4/1997 or 2001-04-23 (various combinations are acceptable, but the month always precedes -the day). -""" % timestr) +the day).""" % timestr) + + # Test for straight integer + if cls._integer_regexp.search(timestr): return int(timestr) # Test for w3-datetime format, possibly missing tzd t = cls.stringtotime(timestr) or cls.stringtotime(timestr+cls.gettzd()) -- cgit v1.2.1