diff options
Diffstat (limited to 'rdiff-backup/rdiff_backup/increment.py')
| -rw-r--r-- | rdiff-backup/rdiff_backup/increment.py | 422 |
1 files changed, 42 insertions, 380 deletions
diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py index a11dd6a..a9d5413 100644 --- a/rdiff-backup/rdiff_backup/increment.py +++ b/rdiff-backup/rdiff_backup/increment.py @@ -19,14 +19,11 @@ """Provides functions and *ITR classes, for writing increment files""" -import traceback -from log import Log -import Globals, Time, MiscStats, rorpiter, TempFile, robust, \ - statistics, rpath, static, lazy, Rdiff, Hardlink +import Globals, Time, rpath, Rdiff, log, statistics -def Increment_action(new, mirror, incpref): - """Main file incrementing function, returns robust.Action +def Increment(new, mirror, incpref): + """Main file incrementing function, returns inc file created new is the file on the active partition, mirror is the mirrored file from the last backup, @@ -35,70 +32,57 @@ def Increment_action(new, mirror, incpref): This function basically moves the information about the mirror file to incpref. - The returned robust.Action when executed should return the name - of the incfile, or None if none was created. - """ if not (new and new.lstat() or mirror.lstat()): - return robust.null_action # Files deleted in meantime, do nothing + return None # Files deleted in meantime, do nothing - Log("Incrementing mirror file " + mirror.path, 5) + log.Log("Incrementing mirror file " + mirror.path, 5) if ((new and new.isdir()) or mirror.isdir()) and not incpref.isdir(): incpref.mkdir() - if not mirror.lstat(): return makemissing_action(incpref) - elif mirror.isdir(): return makedir_action(mirror, incpref) + if not mirror.lstat(): incrp = makemissing(incpref) + elif mirror.isdir(): incrp = makedir(mirror, incpref) elif new.isreg() and mirror.isreg(): - return makediff_action(new, mirror, incpref) - else: return makesnapshot_action(mirror, incpref) - -def Increment(new, mirror, incpref): - return Increment_action(new, mirror, incpref).execute() + incrp = makediff(new, mirror, incpref) + else: incrp = makesnapshot(mirror, incpref) + statistics.process_increment(incrp) + return incrp -def makemissing_action(incpref): +def makemissing(incpref): """Signify that mirror file was missing""" - def final(init_val): - incrp = get_inc_ext(incpref, "missing") - incrp.touch() - return incrp - return robust.Action(None, final, None) + incrp = get_inc_ext(incpref, "missing") + incrp.touch() + return incrp + +def iscompressed(mirror): + """Return true if mirror's increments should be compressed""" + return (Globals.compression and + not Globals.no_compression_regexp.match(mirror.path)) -def makesnapshot_action(mirror, incpref): +def makesnapshot(mirror, incpref): """Copy mirror to incfile, since new is quite different""" - if (mirror.isreg() and Globals.compression and - not Globals.no_compression_regexp.match(mirror.path)): - snapshotrp = get_inc_ext(incpref, "snapshot.gz") - return robust.copy_with_attribs_action(mirror, snapshotrp, 1) - else: - snapshotrp = get_inc_ext(incpref, "snapshot") - return robust.copy_with_attribs_action(mirror, snapshotrp, None) + compress = iscompressed(mirror) + if compress: snapshotrp = get_inc_ext(incpref, "snapshot.gz") + else: snapshotrp = get_inc_ext(incpref, "snapshot") + rpath.copy_with_attribs(mirror, snapshotrp, compress) + return snapshotrp -def makediff_action(new, mirror, incpref): +def makediff(new, mirror, incpref): """Make incfile which is a diff new -> mirror""" - if (Globals.compression and - not Globals.no_compression_regexp.match(mirror.path)): - diff = get_inc_ext(incpref, "diff.gz") - compress = 1 - else: - diff = get_inc_ext(incpref, "diff") - compress = None + compress = iscompressed(mirror) + if compress: diff = get_inc_ext(incpref, "diff.gz") + else: diff = get_inc_ext(incpref, "diff") - diff_tf = TempFile.new(diff) - def init(): - Rdiff.write_delta(new, mirror, diff_tf, compress) - rpath.copy_attribs(mirror, diff_tf) - return diff - return robust.make_tf_robustaction(init, diff_tf, diff) + Rdiff.write_delta(new, mirror, diff, compress) + rpath.copy_attribs(mirror, diff) + return diff -def makedir_action(mirrordir, incpref): +def makedir(mirrordir, incpref): """Make file indicating directory mirrordir has changed""" dirsign = get_inc_ext(incpref, "dir") - tf = TempFile.new(dirsign) - def init(): - tf.touch() - rpath.copy_attribs(mirrordir, tf) - return dirsign - return robust.make_tf_robustaction(init, tf, dirsign) + dirsign.touch() + rpath.copy_attribs(mirrordir, dirsign) + return dirsign def get_inc(rp, time, typestr): """Return increment like rp but with time and typestr suffixes""" @@ -107,344 +91,22 @@ def get_inc(rp, time, typestr): incrp = rp.__class__(rp.conn, rp.base, rp.index[:-1] + (addtostr(rp.index[-1]),)) else: incrp = rp.__class__(rp.conn, addtostr(rp.base), rp.index) - if Globals.quoting_enabled: incrp.quote_path() return incrp -def get_inc_ext(rp, typestr): - """Return increment with specified type and correct time +def get_inc_ext(rp, typestr, inctime = None): + """Return increment with specified type and time t If the file exists, then probably a previous backup has been aborted. We then keep asking FindTime to get a time later than the one that already has an inc file. """ - inctime = 0 + if inctime is None: inctime = Time.prevtime while 1: - #inctime = robust.Resume.FindTime(rp.index, inctime) - inctime = Time.prevtime incrp = get_inc(rp, inctime, typestr) if not incrp.lstat(): break else: - assert 0, "Inc file already present" + inctime += 1 + log.Log("Warning, increment %s already exists" % (incrp.path,), 2) return incrp - -class IncrementITRB(statistics.ITRB): - """Patch and increment mirror directory - - This has to be an ITR because directories that have files in them - changed are flagged with an increment marker. There are four - possibilities as to the order: - - 1. Normal file -> Normal file: right away - 2. Directory -> Directory: wait until files in the directory - are processed, as we won't know whether to add a marker - until the end. - 3. Normal file -> Directory: right away, so later files will - have a directory to go into. - 4. Directory -> Normal file: Wait until the end, so we can - process all the files in the directory. - - """ - # Iff true, mirror file was a directory - mirror_isdirectory = None - # If set, what the directory on the mirror side will be replaced with - directory_replacement = None - # True iff there has been some change at this level or lower (used - # for marking directories to be flagged) - changed = None - # Holds the RPath of the created increment file, if any - incrp = None - - def __init__(self, inc_rpath): - """Set inc_rpath, an rpath of the base of the tree""" - self.inc_rpath = inc_rpath - statistics.ITRB.__init__(self) - - def start_process(self, index, diff_rorp, dsrp): - """Initial processing of file - - diff_rorp is the RORPath of the diff from the remote side, and - dsrp is the local file to be incremented - - """ - self.start_stats(dsrp) - incpref = self.inc_rpath.new_index(index) - if Globals.quoting_enabled: incpref.quote_path() - if dsrp.isdir(): - self.init_dir(dsrp, diff_rorp, incpref) - self.mirror_isdirectory = 1 - else: self.init_non_dir(dsrp, diff_rorp, incpref) - self.setvals(diff_rorp, dsrp, incpref) - - def override_changed(self): - """Set changed flag to true - - This is used only at the top level of a backup, to make sure - that a marker is created recording every backup session. - - """ - self.changed = 1 - - def setvals(self, diff_rorp, dsrp, incpref): - """Record given values in state dict since in directory - - We don't do these earlier in case of a problem inside the - init_* functions. Index isn't given because it is done by the - superclass. - - """ - self.diff_rorp = diff_rorp - self.dsrp = dsrp - self.incpref = incpref - - def init_dir(self, dsrp, diff_rorp, incpref): - """Process a directory (initial pass) - - If the directory is changing into a normal file, we need to - save the normal file data in a temp file, and then create the - real file once we are done with everything inside the - directory. - - """ - if not (incpref.lstat() and incpref.isdir()): incpref.mkdir() - if diff_rorp and diff_rorp.isreg() and diff_rorp.file: - tf = TempFile.new(dsrp) - def init(): - rpath.copy_with_attribs(diff_rorp, tf) - tf.set_attached_filetype(diff_rorp.get_attached_filetype()) - def error(exc, ran_init, init_val): tf.delete() - robust.Action(init, None, error).execute() - self.directory_replacement = tf - - def init_non_dir(self, dsrp, diff_rorp, incpref): - """Process a non directory file (initial pass)""" - if not diff_rorp: return # no diff, so no change necessary - if diff_rorp.isreg() and (dsrp.isreg() or diff_rorp.isflaglinked()): - # Write updated mirror to temp file so we can compute - # reverse diff locally - mirror_tf = TempFile.new(dsrp) - old_dsrp_tf = TempFile.new(dsrp) - def init_thunk(): - if diff_rorp.isflaglinked(): - Hardlink.link_rp(diff_rorp, mirror_tf, dsrp) - else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp, - mirror_tf).execute() - self.incrp = Increment_action(mirror_tf, dsrp, - incpref).execute() - if dsrp.lstat(): rpath.rename(dsrp, old_dsrp_tf) - mirror_tf.rename(dsrp) - - def final(init_val): old_dsrp_tf.delete() - def error(exc, ran_init, init_val): - if ran_init: old_dsrp_tf.delete() # everything is fine - else: # restore to previous state - if old_dsrp_tf.lstat(): old_dsrp_tf.rename(dsrp) - if self.incrp: self.incrp.delete() - mirror_tf.delete() - - robust.Action(init_thunk, final, error).execute() - else: self.incrp = robust.chain( - Increment_action(diff_rorp, dsrp, incpref), - rorpiter.patchonce_action(None, dsrp, diff_rorp)).execute()[0] - - self.changed = 1 - - def end_process(self): - """Do final work when leaving a tree (directory)""" - diff_rorp, dsrp, incpref = self.diff_rorp, self.dsrp, self.incpref - if (self.mirror_isdirectory and (diff_rorp or self.changed) - or self.directory_replacement): - if self.directory_replacement: - tf = self.directory_replacement - self.incrp = robust.chain( - Increment_action(tf, dsrp, incpref), - rorpiter.patchonce_action(None, dsrp, tf)).execute()[0] - tf.delete() - else: - self.incrp = Increment(diff_rorp, dsrp, incpref) - if diff_rorp: - rorpiter.patchonce_action(None, dsrp, diff_rorp).execute() - - self.end_stats(diff_rorp, dsrp, self.incrp) - if self.mirror_isdirectory or dsrp.isdir(): - MiscStats.write_dir_stats_line(self, dsrp.index) - - def can_fast_process(self, index, diff_rorp, dsrp): - """True if there is no change in file and is just a leaf""" - return not diff_rorp and dsrp.isreg() - - def fast_process(self, index, diff_rorp, dsrp): - """Just update statistics""" - statistics.ITRB.fast_process(self, dsrp) - - def branch_process(self, branch): - """Update statistics, and the has_changed flag if change in branch""" - if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio) - if branch.changed: self.changed = 1 - self.add_file_stats(branch) - - -class PatchITRB(statistics.ITRB): - """Patch an rpath with the given diff iters (use with IterTreeReducer) - - The main complication here involves directories. We have to - finish processing the directory after what's in the directory, as - the directory may have inappropriate permissions to alter the - contents or the dir's mtime could change as we change the - contents. - - """ - def __init__(self, basis_root_rp): - """Set basis_root_rp, the base of the tree to be incremented""" - self.basis_root_rp = basis_root_rp - assert basis_root_rp.conn is Globals.local_connection - #statistics.ITRB.__init__(self) - self.dir_replacement, self.dir_update = None, None - self.cached_rp = None - - def get_rp_from_root(self, index): - """Return RPath by adding index to self.basis_root_rp""" - if not self.cached_rp or self.cached_rp.index != index: - self.cached_rp = self.basis_root_rp.new_index(index) - return self.cached_rp - - def can_fast_process(self, index, diff_rorp): - """True if diff_rorp and mirror are not directories""" - rp = self.get_rp_from_root(index) - return not diff_rorp.isdir() and not rp.isdir() - - def fast_process(self, index, diff_rorp): - """Patch base_rp with diff_rorp (case where neither is directory)""" - rp = self.get_rp_from_root(index) - tf = TempFile.new(rp) - self.patch_to_temp(rp, diff_rorp, tf) - tf.rename(rp) - - def patch_to_temp(self, basis_rp, diff_rorp, new): - """Patch basis_rp, writing output in new, which doesn't exist yet""" - if diff_rorp.isflaglinked(): - Hardlink.link_rp(diff_rorp, new, self.basis_root_rp) - elif diff_rorp.get_attached_filetype() == 'snapshot': - rpath.copy(diff_rorp, new) - else: - assert diff_rorp.get_attached_filetype() == 'diff' - Rdiff.patch_local(basis_rp, diff_rorp, new) - if new.lstat(): rpath.copy_attribs(diff_rorp, new) - - def start_process(self, index, diff_rorp): - """Start processing directory - record information for later""" - base_rp = self.base_rp = self.get_rp_from_root(index) - assert diff_rorp.isdir() or base_rp.isdir() - if diff_rorp.isdir(): self.prepare_dir(diff_rorp, base_rp) - else: self.set_dir_replacement(diff_rorp, base_rp) - - def set_dir_replacement(self, diff_rorp, base_rp): - """Set self.dir_replacement, which holds data until done with dir - - This is used when base_rp is a dir, and diff_rorp is not. - - """ - assert diff_rorp.get_attached_filetype() == 'snapshot' - self.dir_replacement = TempFile.new(base_rp) - rpath.copy_with_attribs(diff_rorp, self.dir_replacement) - - def prepare_dir(self, diff_rorp, base_rp): - """Prepare base_rp to turn into a directory""" - self.dir_update = diff_rorp.getRORPath() # make copy in case changes - if not base_rp.isdir(): - if base_rp.lstat(): base_rp.delete() - base_rp.mkdir() - base_rp.chmod(0700) - - def end_process(self): - """Finish processing directory""" - if self.dir_update: - assert self.base_rp.isdir() - rpath.copy_attribs(self.dir_update, self.base_rp) - else: - assert self.dir_replacement and self.base_rp.isdir() - self.base_rp.rmdir() - self.dir_replacement.rename(self.base_rp) - - -class IncrementITRB(PatchITRB): - """Patch an rpath with the given diff iters and write increments - - Like PatchITRB, but this time also write increments. - - """ - def __init__(self, basis_root_rp, inc_root_rp): - self.inc_root_rp = inc_root_rp - self.cached_incrp = None - PatchITRB.__init__(self, basis_root_rp) - - def get_incrp(self, index): - """Return inc RPath by adding index to self.basis_root_rp""" - if not self.cached_incrp or self.cached_incrp.index != index: - self.cached_incrp = self.inc_root_rp.new_index(index) - return self.cached_incrp - - def fast_process(self, index, diff_rorp): - """Patch base_rp with diff_rorp and write increment (neither is dir)""" - rp = self.get_rp_from_root(index) - tf = TempFile.new(rp) - self.patch_to_temp(rp, diff_rorp, tf) - Increment(tf, rp, self.get_incrp(index)) - tf.rename(rp) - - def start_process(self, index, diff_rorp): - """Start processing directory""" - base_rp = self.base_rp = self.get_rp_from_root(index) - assert diff_rorp.isdir() or base_rp.isdir() - if diff_rorp.isdir(): - Increment(diff_rorp, base_rp, self.get_incrp(index)) - self.prepare_dir(diff_rorp, base_rp) - else: - self.set_dir_replacement(diff_rorp, base_rp) - Increment(self.dir_replacement, base_rp, self.get_incrp(index)) - - -class MirrorITRB(statistics.ITRB): - """Like IncrementITR, but only patch mirror directory, don't increment""" - def __init__(self, inc_rpath): - """Set inc_rpath, an rpath of the base of the inc tree""" - self.inc_rpath = inc_rpath - statistics.ITRB.__init__(self) - - def start_process(self, index, diff_rorp, mirror_dsrp): - """Initialize statistics and do actual writing to mirror""" - self.start_stats(mirror_dsrp) - if (diff_rorp and diff_rorp.isdir() or - not diff_rorp and mirror_dsrp.isdir()): - # mirror_dsrp will end up as directory, update attribs later - if not diff_rorp: diff_rorp = mirror_dsrp.get_rorpath() - if not mirror_dsrp.isdir(): - mirror_dsrp.delete() - mirror_dsrp.mkdir() - elif diff_rorp and not diff_rorp.isplaceholder(): - rorpiter.patchonce_action(None, mirror_dsrp, diff_rorp).execute() - - self.incpref = self.inc_rpath.new_index(index) - self.diff_rorp, self.mirror_dsrp = diff_rorp, mirror_dsrp - - def end_process(self): - """Update statistics when leaving""" - self.end_stats(self.diff_rorp, self.mirror_dsrp) - if self.mirror_dsrp.isdir(): - rpath.copy_attribs(self.diff_rorp, self.mirror_dsrp) - MiscStats.write_dir_stats_line(self, self.mirror_dsrp.index) - - def can_fast_process(self, index, diff_rorp, mirror_dsrp): - """True if there is no change in file and it is just a leaf""" - return not diff_rorp and mirror_dsrp.isreg() - - def fast_process(self, index, diff_rorp, mirror_dsrp): - """Just update statistics""" - statistics.ITRB.fast_process(self, mirror_dsrp) - - def branch_process(self, branch): - """Update statistics with subdirectory results""" - if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio) - self.add_file_stats(branch) - |