summaryrefslogtreecommitdiff
path: root/rdiff-backup
diff options
context:
space:
mode:
Diffstat (limited to 'rdiff-backup')
-rw-r--r--rdiff-backup/CHANGELOG3
-rw-r--r--rdiff-backup/TODO11
-rwxr-xr-xrdiff-backup/dist/makedist4
-rw-r--r--rdiff-backup/rdiff-backup.19
-rw-r--r--rdiff-backup/rdiff_backup/FilenameMapping.py6
-rw-r--r--rdiff-backup/rdiff_backup/Globals.py1
-rw-r--r--rdiff-backup/rdiff_backup/Main.py8
-rw-r--r--rdiff-backup/rdiff_backup/backup.py106
-rw-r--r--rdiff-backup/rdiff_backup/increment.py2
-rw-r--r--rdiff-backup/rdiff_backup/longname.py275
-rw-r--r--rdiff-backup/rdiff_backup/metadata.py11
-rw-r--r--rdiff-backup/rdiff_backup/regress.py5
-rw-r--r--rdiff-backup/rdiff_backup/restore.py45
-rw-r--r--rdiff-backup/rdiff_backup/rpath.py40
-rw-r--r--rdiff-backup/testing/commontest.py4
-rw-r--r--rdiff-backup/testing/longnametest.py141
16 files changed, 555 insertions, 116 deletions
diff --git a/rdiff-backup/CHANGELOG b/rdiff-backup/CHANGELOG
index 591f12c..fdea69b 100644
--- a/rdiff-backup/CHANGELOG
+++ b/rdiff-backup/CHANGELOG
@@ -9,6 +9,9 @@ connection is lost.
When removing older than, delete empty increments directories
+Long filename bug finally fixed (phew). rdiff-backup should now
+correctly mirror any file that it can read.
+
New in v1.1.2 (2005/11/06)
--------------------------
diff --git a/rdiff-backup/TODO b/rdiff-backup/TODO
index 9cab567..10bb466 100644
--- a/rdiff-backup/TODO
+++ b/rdiff-backup/TODO
@@ -1,6 +1,13 @@
-For comparing, check source filesystem's abilities
+Make sure remove older than doesn't delete current snapshot. When no
+increments found, don't act like there's an error (no stderr).
+
+Port close file fix to devel version
+
+Regress needs to fix alternate names also
-Clean up connection dropped message
+See if rorpiter.CacheIndexable is really necessary
+
+For comparing, check source filesystem's abilities
Clean up compare reports
diff --git a/rdiff-backup/dist/makedist b/rdiff-backup/dist/makedist
index 07f285d..7f69865 100755
--- a/rdiff-backup/dist/makedist
+++ b/rdiff-backup/dist/makedist
@@ -111,8 +111,8 @@ def MakeTar():
"FilenameMapping.py", "fs_abilities.py",
"Hardlink.py", "hash.py", "increment.py", "__init__.py",
"iterfile.py", "lazy.py", "librsync.py",
- "log.py", "Main.py", "manage.py", "metadata.py",
- "Rdiff.py", "regress.py", "restore.py",
+ "log.py", "longname.py", "Main.py", "manage.py",
+ "metadata.py", "Rdiff.py", "regress.py", "restore.py",
"robust.py", "rorpiter.py", "rpath.py",
"Security.py", "selection.py",
"SetConnections.py", "static.py",
diff --git a/rdiff-backup/rdiff-backup.1 b/rdiff-backup/rdiff-backup.1
index 6f94955..876e203 100644
--- a/rdiff-backup/rdiff-backup.1
+++ b/rdiff-backup/rdiff-backup.1
@@ -961,15 +961,6 @@ automatically (as from
or similar) it is probably a good idea to check the exit code.
.SH BUGS
-rdiff-backup uses the shell command
-.BR mknod (1)
-to backup device files (e.g. /dev/ttyS0), so device files won't be
-handled correctly on systems with non-standard mknod syntax.
-.PP
-Files whose names are close to the maximum length (e.g. 235 chars if
-the maximum is 255) may be skipped because the filenames of related
-increment files would be too long.
-.PP
The gzip library in versions 2.2 and earlier of python (but fixed in
2.3a1) has trouble producing files over 2GB in length. This bug will
prevent rdiff-backup from producing large compressed increments
diff --git a/rdiff-backup/rdiff_backup/FilenameMapping.py b/rdiff-backup/rdiff_backup/FilenameMapping.py
index 066b86a..b730fd3 100644
--- a/rdiff-backup/rdiff_backup/FilenameMapping.py
+++ b/rdiff-backup/rdiff_backup/FilenameMapping.py
@@ -32,8 +32,6 @@ handle that error.)
import re, types
import Globals, log, rpath
-max_filename_length = 255
-
# If true, enable character quoting, and set characters making
# regex-style range.
chars_to_quote = None
@@ -114,8 +112,8 @@ class QuotedRPath(rpath.RPath):
"""
def __init__(self, connection, base, index = (), data = None):
"""Make new QuotedRPath"""
- quoted_index = tuple(map(quote, index))
- rpath.RPath.__init__(self, connection, base, quoted_index, data)
+ self.quoted_index = tuple(map(quote, index))
+ rpath.RPath.__init__(self, connection, base, self.quoted_index, data)
self.index = index
def listdir(self):
diff --git a/rdiff-backup/rdiff_backup/Globals.py b/rdiff-backup/rdiff_backup/Globals.py
index f440dbd..253c8a7 100644
--- a/rdiff-backup/rdiff_backup/Globals.py
+++ b/rdiff-backup/rdiff_backup/Globals.py
@@ -221,7 +221,6 @@ never_drop_acls = None
# prevent highbit permissions on systems which don't support them.)
permission_mask = 07777
-
def get(name):
"""Return the value of something in this module"""
return globals()[name]
diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py
index ce29f80..54ae90a 100644
--- a/rdiff-backup/rdiff_backup/Main.py
+++ b/rdiff-backup/rdiff_backup/Main.py
@@ -74,10 +74,10 @@ def parse_cmdlineoptions(arglist):
"include-globbing-filelist-stdin", "include-regexp=",
"include-special-files", "include-symbolic-links",
"list-at-time=", "list-changed-since=", "list-increments",
- "list-increment-sizes", "never-drop-acls", "no-acls",
- "no-carbonfile", "no-compare-inode", "no-compression",
- "no-compression-regexp=", "no-eas", "no-file-statistics",
- "no-hard-links", "null-separator",
+ "list-increment-sizes", "never-drop-acls",
+ "no-acls", "no-carbonfile",
+ "no-compare-inode", "no-compression", "no-compression-regexp=",
+ "no-eas", "no-file-statistics", "no-hard-links", "null-separator",
"override-chars-to-quote=", "parsable-output",
"preserve-numerical-ids", "print-statistics",
"remote-cmd=", "remote-schema=",
diff --git a/rdiff-backup/rdiff_backup/backup.py b/rdiff-backup/rdiff_backup/backup.py
index dfe21ac..31d450d 100644
--- a/rdiff-backup/rdiff_backup/backup.py
+++ b/rdiff-backup/rdiff_backup/backup.py
@@ -23,7 +23,7 @@ from __future__ import generators
import errno
import Globals, metadata, rorpiter, TempFile, Hardlink, robust, increment, \
rpath, static, log, selection, Time, Rdiff, statistics, iterfile, \
- hash
+ hash, longname
def Mirror(src_rpath, dest_rpath):
"""Turn dest_rpath into a copy of src_rpath"""
@@ -189,11 +189,12 @@ class DestinationStruct:
elif dest_rorp:
dest_sig = dest_rorp.getRORPath()
if dest_rorp.isreg():
- sig_fp = cls.get_one_sig_fp(dest_base_rpath.new_index(index))
+ dest_rp = longname.get_mirror_rp(dest_base_rpath, dest_rorp)
+ sig_fp = cls.get_one_sig_fp(dest_rp)
if sig_fp is None: return None
dest_sig.setfile(sig_fp)
else: dest_sig = rpath.RORPath(index)
- return dest_sig
+ return dest_sig
def get_one_sig_fp(cls, dest_rp):
"""Return a signature fp of given index, corresponding to reg file"""
@@ -467,44 +468,26 @@ class PatchITRB(rorpiter.ITRBranch):
self.statfileobj = (statistics.get_active_statfileobj() or
statistics.StatFileObj())
self.dir_replacement, self.dir_update = None, None
- self.cached_rp = None
self.CCPP = CCPP
self.error_handler = robust.get_error_handler("UpdateError")
- def get_rp_from_root(self, index):
- """Return RPath by adding index to self.basis_root_rp"""
- if not self.cached_rp or self.cached_rp.index != index:
- self.cached_rp = self.basis_root_rp.new_index(index)
- return self.cached_rp
-
- def check_long_name(self, func, *args):
- """Execute function, checking for ENAMETOOLONG error"""
- try: result = func(*args)
- except OSError, exc:
- if (errno.errorcode.has_key(exc[0]) and
- errno.errorcode[exc[0]] == 'ENAMETOOLONG'):
- self.error_handler(exc, args[0])
- return None
- else: raise
- return result
-
def can_fast_process(self, index, diff_rorp):
"""True if diff_rorp and mirror are not directories"""
- rp = self.check_long_name(self.get_rp_from_root, index)
- # filename too long error qualifies (hack)
- return not rp or (not diff_rorp.isdir() and not rp.isdir())
+ mirror_rorp = self.CCPP.get_mirror_rorp(index)
+ return not (diff_rorp.isdir() or (mirror_rorp and mirror_rorp.isdir()))
def fast_process(self, index, diff_rorp):
"""Patch base_rp with diff_rorp (case where neither is directory)"""
- rp = self.check_long_name(self.get_rp_from_root, index)
- if not rp: return
- tf = TempFile.new(rp)
- if self.patch_to_temp(rp, diff_rorp, tf):
+ mirror_rp, discard = longname.get_mirror_inc_rps(
+ self.CCPP.get_rorps(index), self.basis_root_rp)
+ assert not mirror_rp.isdir(), mirror_rp
+ tf = TempFile.new(mirror_rp)
+ if self.patch_to_temp(mirror_rp, diff_rorp, tf):
if tf.lstat():
- rpath.rename(tf, rp)
+ rpath.rename(tf, mirror_rp)
self.CCPP.flag_success(index)
- elif rp.lstat():
- rp.delete()
+ elif mirror_rp and mirror_rp.lstat():
+ mirror_rp.delete()
self.CCPP.flag_deleted(index)
else:
tf.setdata()
@@ -583,11 +566,12 @@ class PatchITRB(rorpiter.ITRBranch):
def start_process(self, index, diff_rorp):
"""Start processing directory - record information for later"""
- base_rp = self.base_rp = self.get_rp_from_root(index)
- assert diff_rorp.isdir() or base_rp.isdir() or not base_rp.index
- if diff_rorp.isdir(): self.prepare_dir(diff_rorp, base_rp)
- elif self.set_dir_replacement(diff_rorp, base_rp):
- self.CCPP.flag_success(index)
+ self.base_rp, discard = longname.get_mirror_inc_rps(
+ self.CCPP.get_rorps(index), self.basis_root_rp)
+ if diff_rorp.isdir(): self.prepare_dir(diff_rorp, self.base_rp)
+ elif self.set_dir_replacement(diff_rorp, self.base_rp):
+ if diff_rorp.lstat(): self.CCPP.flag_success(index)
+ else: self.CCPP.flag_deleted(index)
def set_dir_replacement(self, diff_rorp, base_rp):
"""Set self.dir_replacement, which holds data until done with dir
@@ -607,10 +591,11 @@ class PatchITRB(rorpiter.ITRBranch):
else: return 1
def prepare_dir(self, diff_rorp, base_rp):
- """Prepare base_rp to turn into a directory"""
+ """Prepare base_rp to be a directory"""
self.dir_update = diff_rorp.getRORPath() # make copy in case changes
if not base_rp.isdir():
- if base_rp.lstat(): base_rp.delete()
+ if base_rp.lstat(): self.base_rp.delete()
+ base_rp.setdata()
base_rp.mkdir()
self.CCPP.flag_success(diff_rorp.index)
else: # maybe no change, so query CCPP before tagging success
@@ -622,8 +607,7 @@ class PatchITRB(rorpiter.ITRBranch):
if self.dir_update:
assert self.base_rp.isdir()
rpath.copy_attribs(self.dir_update, self.base_rp)
- else:
- assert self.dir_replacement
+ elif self.dir_replacement:
self.base_rp.rmdir()
if self.dir_replacement.lstat():
rpath.rename(self.dir_replacement, self.base_rp)
@@ -637,32 +621,24 @@ class IncrementITRB(PatchITRB):
"""
def __init__(self, basis_root_rp, inc_root_rp, rorp_cache):
self.inc_root_rp = inc_root_rp
- self.cached_incrp = None
PatchITRB.__init__(self, basis_root_rp, rorp_cache)
- def get_incrp(self, index):
- """Return inc RPath by adding index to self.basis_root_rp"""
- if not self.cached_incrp or self.cached_incrp.index != index:
- self.cached_incrp = self.inc_root_rp.new_index(index)
- return self.cached_incrp
-
def fast_process(self, index, diff_rorp):
"""Patch base_rp with diff_rorp and write increment (neither is dir)"""
- rp = self.check_long_name(self.get_rp_from_root, index)
- if not rp: return
- tf = TempFile.new(rp)
- if self.patch_to_temp(rp, diff_rorp, tf):
- inc = self.check_long_name(increment.Increment,
- tf, rp, self.get_incrp(index))
+ mirror_rp, inc_prefix = longname.get_mirror_inc_rps(
+ self.CCPP.get_rorps(index), self.basis_root_rp, self.inc_root_rp)
+ tf = TempFile.new(mirror_rp)
+ if self.patch_to_temp(mirror_rp, diff_rorp, tf):
+ inc = increment.Increment(tf, mirror_rp, inc_prefix)
if inc is not None:
self.CCPP.set_inc(index, inc)
if inc.isreg():
inc.fsync_with_dir() # Write inc before rp changed
if tf.lstat():
- rpath.rename(tf, rp)
+ rpath.rename(tf, mirror_rp)
self.CCPP.flag_success(index)
- elif rp.lstat():
- rp.delete()
+ elif mirror_rp.lstat():
+ mirror_rp.delete()
self.CCPP.flag_deleted(index)
return # normal return, otherwise error occurred
tf.setdata()
@@ -670,17 +646,19 @@ class IncrementITRB(PatchITRB):
def start_process(self, index, diff_rorp):
"""Start processing directory"""
- base_rp = self.base_rp = self.get_rp_from_root(index)
- assert diff_rorp.isdir() or base_rp.isdir()
+ self.base_rp, inc_prefix = longname.get_mirror_inc_rps(
+ self.CCPP.get_rorps(index), self.basis_root_rp, self.inc_root_rp)
+ self.base_rp.setdata()
+ assert diff_rorp.isdir() or self.base_rp.isdir()
if diff_rorp.isdir():
- inc = self.check_long_name(increment.Increment,
- diff_rorp, base_rp, self.get_incrp(index))
+ inc = increment.Increment(diff_rorp, self.base_rp, inc_prefix)
if inc and inc.isreg():
inc.fsync_with_dir() # must write inc before rp changed
- self.prepare_dir(diff_rorp, base_rp)
- elif self.set_dir_replacement(diff_rorp, base_rp):
- inc = self.check_long_name(increment.Increment,
- self.dir_replacement, base_rp, self.get_incrp(index))
+ self.base_rp.setdata() # in case written by increment above
+ self.prepare_dir(diff_rorp, self.base_rp)
+ elif self.set_dir_replacement(diff_rorp, self.base_rp):
+ inc = increment.Increment(self.dir_replacement, self.base_rp,
+ inc_prefix)
if inc:
self.CCPP.set_inc(index, inc)
self.CCPP.flag_success(index)
diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py
index 1432f76..f1cdbc8 100644
--- a/rdiff-backup/rdiff_backup/increment.py
+++ b/rdiff-backup/rdiff_backup/increment.py
@@ -34,7 +34,7 @@ def Increment(new, mirror, incpref):
"""
log.Log("Incrementing mirror file " + mirror.path, 5)
- if ((new and new.isdir()) or mirror.isdir()) and not incpref.isdir():
+ if ((new and new.isdir()) or mirror.isdir()) and not incpref.lstat():
incpref.mkdir()
if not mirror.lstat(): incrp = makemissing(incpref)
diff --git a/rdiff-backup/rdiff_backup/longname.py b/rdiff-backup/rdiff_backup/longname.py
new file mode 100644
index 0000000..9d3697a
--- /dev/null
+++ b/rdiff-backup/rdiff_backup/longname.py
@@ -0,0 +1,275 @@
+# Copyright 2005 Ben Escoto
+#
+# This file is part of rdiff-backup.
+#
+# rdiff-backup is free software; you can redistribute it and/or modify
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# rdiff-backup is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with rdiff-backup; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+
+"""Handle long filenames
+
+rdiff-backup sometimes wants to write filenames longer than allowed by
+the destination directory. This can happen in 3 ways:
+
+1) Because the destination directory has a low maximum length limit.
+2) When the source directory has a filename close to the limit, so
+ that its increments would be above the limit.
+3) When quoting is enabled, so that even the mirror filenames are too
+ long.
+
+When rdiff-backup would otherwise write a file whose name is too long,
+instead it either skips the operation altogether (for non-regular
+files), or writes the data to a unique file in the
+rdiff-backup-data/long-filename directory. This file will have an
+arbitrary basename, but if it's an increment the suffix will be the
+same. The name will be recorded in the mirror_metadata so we can find
+it later.
+
+"""
+
+import types, errno
+import log, Globals, restore, rpath, FilenameMapping, regress
+
+long_name_dir = "long_filename_data"
+rootrp = None
+
+def get_long_rp(base = None):
+ """Return an rpath in long name directory with given base"""
+ global rootrp
+ if not rootrp:
+ rootrp = Globals.rbdir.append(long_name_dir)
+ if not rootrp.lstat(): rootrp.mkdir()
+ if base: return rootrp.append(base)
+ else: return rootrp
+
+
+# ------------------------------------------------------------------
+# These functions used mainly for backing up
+
+# integer number of next free prefix. Names will be created from
+# integers consecutively like '1', '2', and so on.
+free_name_counter = None
+
+# Filename which holds the next available free name in it
+counter_filename = "next_free"
+
+def get_next_free():
+ """Return next free filename available in the long filename directory"""
+ global free_name_counter
+ def scan_next_free():
+ """Return value of free_name_counter by listing long filename dir"""
+ log.Log("Setting next free from long filenames dir", 5)
+ cur_high = 0
+ for filename in get_long_rp().listdir():
+ try: i = int(filename.split('.')[0])
+ except ValueError: continue
+ if i > cur_high: cur_high = i
+ return cur_high + 1
+
+ def read_next_free():
+ """Return next int free by reading the next_free file, or None"""
+ rp = get_long_rp(counter_filename)
+ if not rp.lstat(): return None
+ return int(rp.get_data())
+
+ def write_next_free(i):
+ """Write value i into the counter file"""
+ rp = get_long_rp(counter_filename)
+ if rp.lstat(): rp.delete()
+ rp.write_string(str(free_name_counter))
+ rp.fsync_with_dir()
+
+ if not free_name_counter: free_name_counter = read_next_free()
+ if not free_name_counter: free_name_counter = scan_next_free()
+ filename = str(free_name_counter)
+ rp = get_long_rp(filename)
+ assert not rp.lstat(), "Unexpected file at %s found" % (rp.path,)
+ free_name_counter += 1
+ write_next_free(free_name_counter)
+ return filename
+
+
+def check_new_index(base, index, make_dirs = 0):
+ """Return new rpath with given index, or None if that is too long
+
+ If make_dir is True, make any parent directories to assure that
+ file is really too long, and not just in directories that don't exist.
+
+ """
+ def wrap_call(func, *args):
+ try: result = func(*args)
+ except EnvironmentError, exc:
+ if (errno.errorcode.has_key(exc[0]) and
+ errno.errorcode[exc[0]] == 'ENAMETOOLONG'):
+ return None
+ raise
+ return result
+
+ def make_parent(rp):
+ parent = rp.get_parent_rp()
+ if parent.lstat(): return 1
+ parent.makedirs()
+ return 2
+
+ rp = wrap_call(base.new_index, index)
+ if not make_dirs or not rp or rp.lstat(): return rp
+
+ parent_result = wrap_call(make_parent, rp)
+ if not parent_result: return None
+ elif parent_result == 1: return rp
+ else: return wrap_call(base.new_index, index)
+
+def get_mirror_rp(mirror_base, mirror_rorp):
+ """Get the mirror_rp for reading a regular file
+
+ This will just be in the mirror_base, unless rorp has an alt
+ mirror name specified. Use new_rorp, unless it is None or empty,
+ and mirror_rorp exists.
+
+ """
+ if mirror_rorp.has_alt_mirror_name():
+ return get_long_rp(mirror_rorp.get_alt_mirror_name())
+ else:
+ rp = check_new_index(mirror_base, mirror_rorp.index)
+ if rp: return rp
+ else: return mirror_base.new_index_empty(index)
+
+def get_mirror_inc_rps(rorp_pair, mirror_root, inc_root = None):
+ """Get (mirror_rp, inc_rp) pair, possibly making new longname base
+
+ To test inc_rp, pad incbase with 50 random (non-quoted) characters
+ and see if that raises an error.
+
+ """
+ if not inc_root: # make fake inc_root if not available
+ inc_root = mirror_root.append_path('rdiff-backup-data/increments')
+
+ def mir_triple_old(old_rorp):
+ """Return (mirror_rp, alt_mirror, alt_inc) from old_rorp"""
+ if old_rorp.has_alt_mirror_name():
+ alt_mirror = old_rorp.get_alt_mirror_name()
+ return (get_long_rp(alt_mirror), alt_mirror, None)
+ else:
+ mirror_rp = mirror_root.new_index(old_rorp.index)
+ if old_rorp.has_alt_inc_name():
+ return (mirror_rp, None, old_rorp.get_alt_inc_name())
+ else: return (mirror_rp, None, None)
+
+ def mir_triple_new(new_rorp):
+ """Return (mirror_rp, alt_mirror, None) from new_rorp"""
+ mirror_rp = check_new_index(mirror_root, new_rorp.index)
+ if mirror_rp: return (mirror_rp, None, None)
+ alt_mirror = get_next_free()
+ return (get_long_rp(alt_mirror), alt_mirror, None)
+
+ def update_rorp(new_rorp, alt_mirror, alt_inc):
+ """Update new_rorp with alternate mirror/inc information"""
+ if not new_rorp or not new_rorp.lstat(): return
+ if alt_mirror: new_rorp.set_alt_mirror_name(alt_mirror)
+ elif alt_inc: new_rorp.set_alt_inc_name(alt_inc)
+
+ def find_inc_pair(index, mirror_rp, alt_mirror, alt_inc):
+ """Return (alt_inc, inc_rp) pair"""
+ if alt_mirror: return (None, mirror_rp)
+ elif alt_inc: return (alt_inc, get_long_rp(alt_inc))
+ elif not index: return (None, inc_root)
+
+ trial_inc_index = index[:-1] + (index[-1] + ('a'*50),)
+ if check_new_index(inc_root, trial_inc_index, make_dirs = 1):
+ return (None, inc_root.new_index(index))
+ alt_inc = get_next_free()
+ return (alt_inc, get_long_rp(alt_inc))
+
+ (new_rorp, old_rorp) = rorp_pair
+ if old_rorp and old_rorp.lstat():
+ mirror_rp, alt_mirror, alt_inc = mir_triple_old(old_rorp)
+ index = old_rorp.index
+ else:
+ assert new_rorp and new_rorp.lstat(), (old_rorp, new_rorp)
+ mirror_rp, alt_mirror, alt_inc = mir_triple_new(new_rorp)
+ index = new_rorp.index
+
+ alt_inc, inc_rp = find_inc_pair(index, mirror_rp, alt_mirror, alt_inc)
+ update_rorp(new_rorp, alt_mirror, alt_inc)
+ return mirror_rp, inc_rp
+
+
+# ------------------------------------------------------------------
+# The following section is for restoring
+
+# This holds a dictionary {incbase: inclist}. The keys are increment
+# bases like '1' or '23', and the values are lists containing the
+# associated increments.
+restore_inc_cache = None
+
+def set_restore_cache():
+ """Initialize restore_inc_cache based on long filename dir"""
+ global restore_inc_cache
+ restore_inc_cache = {}
+ root_rf = restore.RestoreFile(get_long_rp(), get_long_rp(), [])
+ for incbase_rp, inclist in root_rf.yield_inc_complexes(get_long_rp()):
+ restore_inc_cache[incbase_rp.index[-1]] = inclist
+
+def get_inclist(inc_base_name):
+ if not restore_inc_cache: set_restore_cache()
+ try: return restore_inc_cache[inc_base_name]
+ except KeyError: return []
+
+def update_rf(rf, rorp, mirror_root):
+ """Return new or updated restorefile based on alt name info in rorp"""
+ def update_incs(rf, inc_base):
+ """Swap inclist in rf with those with base inc_base and return"""
+ log.Log("Restoring with increment base %s for file %s" %
+ (inc_base, rorp.get_indexpath()), 6)
+ rf.inc_rp = get_long_rp(inc_base)
+ rf.inc_list = get_inclist(inc_base)
+ rf.set_relevant_incs()
+
+ def update_existing_rf(rf, rorp):
+ """Update rf based on rorp, don't make new one"""
+ if rorp.has_alt_mirror_name():
+ inc_name = rorp.get_alt_mirror_name()
+ rf.mirror_rp = get_long_rp(mirror_name)
+ elif rorp.has_alt_inc_name(): inc_name = rorp.get_alt_inc_name()
+ else: inc_name = None
+
+ if inc_name: update_incs(rf, inc_name)
+
+ def make_new_rf(rorp, mirror_root):
+ """Make a new rf when long name info is available"""
+ if rorp.has_alt_mirror_name():
+ inc_name = rorp.get_alt_mirror_name()
+ mirror_rp = get_long_rp(mirror_name)
+ elif rorp.has_alt_inc_name():
+ inc_name = rorp.get_alt_inc_name()
+ mirror_rp = mirror_root.new_index(rorp.index)
+ else: assert 0, "Making new rf when rorp has no alternate name info"
+
+ rf = restore.RestoreFile(mirror_rp, None, [])
+ update_incs(rf, inc_name)
+ return rf
+
+ if not rorp: return rf
+ if rf and not rorp.has_alt_mirror_name() and not rorp.has_alt_inc_name():
+ return rf # Most common case
+ if rf:
+ update_existing_rf(rf, rorp)
+ return rf
+ else: return make_new_rf(rorp, mirror_root)
+
+def update_regressfile(rf, rorp, mirror_root):
+ """Like update_rf except return a regress file object"""
+ rf = update_rf(rf, rorp, mirror_root)
+ if isinstance(rf, regress.RegressFile): return rf
+ return regress.RegressFile(rf.mirror_rp, rf.inc_rp, rf.inc_list)
diff --git a/rdiff-backup/rdiff_backup/metadata.py b/rdiff-backup/rdiff_backup/metadata.py
index db8106b..1f036b3 100644
--- a/rdiff-backup/rdiff_backup/metadata.py
+++ b/rdiff-backup/rdiff_backup/metadata.py
@@ -146,6 +146,15 @@ def RORP2Record(rorpath):
str_list.append(" Gid %s\n" % gid)
str_list.append(" Gname %s\n" % (rorpath.getgname() or ":"))
str_list.append(" Permissions %s\n" % rorpath.getperms())
+
+ # Add long filename information
+ if rorpath.has_alt_mirror_name():
+ str_list.append(" AlternateMirrorName %s\n" %
+ (rorpath.get_alt_mirror_name(),))
+ elif rorpath.has_alt_inc_name():
+ str_list.append(" AlternateIncrementName %s\n" %
+ (rorpath.get_alt_inc_name(),))
+
return "".join(str_list)
line_parsing_regexp = re.compile("^ *([A-Za-z0-9]+) (.+)$", re.M)
@@ -188,6 +197,8 @@ def Record2RORP(record_string):
if data == ':' or data == 'None': data_dict['gname'] = None
else: data_dict['gname'] = data
elif field == "Permissions": data_dict['perms'] = int(data)
+ elif field == "AlternateMirrorName": data_dict['mirrorname'] = data
+ elif field == "AlternateIncrementName": data_dict['incname'] = data
else: raise ParsingError("Unknown field in line '%s %s'" %
(field, data))
return rpath.RORPath(index, data_dict)
diff --git a/rdiff-backup/rdiff_backup/regress.py b/rdiff-backup/rdiff_backup/regress.py
index dfe4796..aac4fd1 100644
--- a/rdiff-backup/rdiff_backup/regress.py
+++ b/rdiff-backup/rdiff_backup/regress.py
@@ -1,4 +1,4 @@
-# Copyright 2002 Ben Escoto
+# Copyright 2002, 2005 Ben Escoto
#
# This file is part of rdiff-backup.
#
@@ -35,7 +35,7 @@ be recovered.
from __future__ import generators
import Globals, restore, log, rorpiter, TempFile, metadata, rpath, C, \
- Time, backup, robust
+ Time, backup, robust, longname
# regress_time should be set to the time we want to regress back to
# (usually the time of the last successful backup)
@@ -193,6 +193,7 @@ def iterate_meta_rfs(mirror_rp, inc_rp):
raw_rfs = iterate_raw_rfs(mirror_rp, inc_rp)
collated = rorpiter.Collate2Iters(raw_rfs, yield_metadata())
for raw_rf, metadata_rorp in collated:
+ raw_rf = longname.update_regressfile(raw_rf, metadata_rorp, mirror_rp)
if not raw_rf:
log.Log("Warning, metadata file has entry for %s,\n"
"but there are no associated files." %
diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py
index 8079511..164e967 100644
--- a/rdiff-backup/rdiff_backup/restore.py
+++ b/rdiff-backup/rdiff_backup/restore.py
@@ -1,4 +1,4 @@
-# Copyright 2002, 2003, 2004 Ben Escoto
+# Copyright 2002, 2003, 2004, 2005 Ben Escoto
#
# This file is part of rdiff-backup.
#
@@ -21,9 +21,7 @@
from __future__ import generators
import tempfile, os, cStringIO
-import Globals, Time, Rdiff, Hardlink, rorpiter, selection, rpath, \
- log, static, robust, metadata, statistics, TempFile, hash
-
+import static, rorpiter
class RestoreError(Exception): pass
@@ -256,7 +254,7 @@ class MirrorStruct:
mir_rorp.flaglinked(Hardlink.get_link_index(mir_rorp))
elif mir_rorp.isreg():
expanded_index = cls.mirror_base.index + mir_rorp.index
- file_fp = cls.rf_cache.get_fp(expanded_index)
+ file_fp = cls.rf_cache.get_fp(expanded_index, mir_rorp)
mir_rorp.setfile(hash.FileWrapper(file_fp))
mir_rorp.set_attached_filetype('snapshot')
return mir_rorp
@@ -325,7 +323,7 @@ class CachedRF:
return "\n".join((s1, s2, s3))
def get_rf(self, index):
- """Return RestoreFile of given index, or None"""
+ """Get a RestoreFile for given index, or None"""
while 1:
if not self.rf_list:
if not self.add_rfs(index): return None
@@ -341,15 +339,16 @@ class CachedRF:
self.add_rfs(index)): return None
else: del self.rf_list[0]
- def get_fp(self, index):
+ def get_fp(self, index, mir_rorp):
"""Return the file object (for reading) of given index"""
- rf = self.get_rf(index)
+ rf = longname.update_rf(self.get_rf(index), mir_rorp,
+ self.root_rf.mirror_rp)
if not rf:
- log.Log("""Error: Unable to retrieve data for file %s!
-The cause is probably data loss from the backup repository.""" %
- (index and "/".join(index) or '.',), 2)
+ log.Log("Error: Unable to retrieve data for file %s!\nThe "
+ "cause is probably data loss from the backup repository."
+ % (index and "/".join(index) or '.',), 2)
return cStringIO.StringIO('')
- return self.get_rf(index).get_restore_fp()
+ return rf.get_restore_fp()
def add_rfs(self, index):
"""Given index, add the rfs in that same directory
@@ -364,9 +363,7 @@ The cause is probably data loss from the backup repository.""" %
temp_rf = RestoreFile(self.root_rf.mirror_rp.new_index(parent_index),
self.root_rf.inc_rp.new_index(parent_index), [])
new_rfs = list(temp_rf.yield_sub_rfs())
- if not new_rfs:
- log.Log("Warning: No RFs added for index %s" % (index,), 2)
- return 0
+ if not new_rfs: return 0
self.rf_list[0:0] = new_rfs
return 1
@@ -384,9 +381,6 @@ class RestoreFile:
"""
def __init__(self, mirror_rp, inc_rp, inc_list):
- assert mirror_rp.index == inc_rp.index, \
- ("mirror and inc indicies don't match: %s %s" %
- (mirror_rp.get_indexpath(), inc_rp.get_indexpath()))
self.index = mirror_rp.index
self.mirror_rp = mirror_rp
self.inc_rp, self.inc_list = inc_rp, inc_list
@@ -485,7 +479,7 @@ constructed from existing increments because last increment had type
%s. Instead of the actual file's data, an empty length file will be
created. This error is probably caused by data loss in the
rdiff-backup destination directory, or a bug in rdiff-backup""" %
- (self.mirror_rp.path, self.relevant_incs[-1].lstat()), 2)
+ (self.mirror_rp.get_indexpath(), self.relevant_incs[-1].lstat()), 2)
return cStringIO.StringIO('')
return robust.check_common_error(error_handler, get_fp)
@@ -505,13 +499,7 @@ rdiff-backup destination directory, or a bug in rdiff-backup""" %
def yield_sub_rfs(self):
"""Return RestoreFiles under current RestoreFile (which is dir)"""
- if not self.mirror_rp.isdir() and not self.inc_rp.isdir():
- log.Log("""Warning: directory %s seems to be missing from backup!
-
-This is probably due to files being deleted manually from the
-rdiff-backup destination directory. In general you shouldn't do this,
-as data loss may result.\n""" % (self.mirror_rp.get_indexpath(),), 2)
- return
+ if not self.mirror_rp.isdir() and not self.inc_rp.isdir(): return
if self.mirror_rp.isdir():
mirror_iter = self.yield_mirrorrps(self.mirror_rp)
else: mirror_iter = iter([])
@@ -743,3 +731,8 @@ class PermissionChanger:
def finish(self):
"""Restore any remaining rps"""
for index, rp, perms in self.open_index_list: rp.chmod(perms)
+
+
+import Globals, Time, Rdiff, Hardlink, selection, rpath, \
+ log, robust, metadata, statistics, TempFile, hash, longname
+
diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py
index 47f3b3e..1f10f1a 100644
--- a/rdiff-backup/rdiff_backup/rpath.py
+++ b/rdiff-backup/rdiff_backup/rpath.py
@@ -357,8 +357,10 @@ class RORPath:
elif key == 'resourcefork' and not Globals.resource_forks_write:
pass
elif key == 'sha1': pass # one or other may not have set
+ elif key == 'mirrorname' or key == 'incname': pass
elif (not other.data.has_key(key) or
- self.data[key] != other.data[key]): return 0
+ self.data[key] != other.data[key]):
+ return 0
if self.lstat() and not self.issym() and Globals.change_ownership:
# Now compare ownership. Symlinks don't have ownership
@@ -654,6 +656,42 @@ class RORPath:
"""Record resource fork in dictionary. Does not write"""
self.data['resourcefork'] = rfork
+ def has_alt_mirror_name(self):
+ """True if rorp has an alternate mirror name specified"""
+ return self.data.has_key('mirrorname')
+
+ def get_alt_mirror_name(self):
+ """Return alternate mirror name (for long filenames)"""
+ return self.data['mirrorname']
+
+ def set_alt_mirror_name(self, filename):
+ """Set alternate mirror name to filename
+
+ Instead of writing to the traditional mirror file, store
+ mirror information in filename in the long filename
+ directory.
+
+ """
+ self.data['mirrorname'] = filename
+
+ def has_alt_inc_name(self):
+ """True if rorp has an alternate increment base specified"""
+ return self.data.has_key('incname')
+
+ def get_alt_inc_name(self):
+ """Return alternate increment base (used for long name support)"""
+ return self.data['incname']
+
+ def set_alt_inc_name(self, name):
+ """Set alternate increment name to name
+
+ If set, increments will be in the long name directory with
+ name as their base. If the alt mirror name is set, this
+ should be set to the same.
+
+ """
+ self.data['incname'] = name
+
def has_sha1(self):
"""True iff self has its sha1 digest set"""
return self.data.has_key('sha1')
diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py
index 27c0406..04cdb93 100644
--- a/rdiff-backup/testing/commontest.py
+++ b/rdiff-backup/testing/commontest.py
@@ -1,10 +1,12 @@
"""commontest - Some functions and constants common to several test cases"""
import os, sys, code
+# Avoid circularities
from rdiff_backup.log import Log
from rdiff_backup.rpath import RPath
from rdiff_backup import Globals, Hardlink, SetConnections, Main, \
selection, lazy, Time, rpath, eas_acls, rorpiter, Security
+
RBBin = "../rdiff-backup"
SourceDir = "../rdiff_backup"
AbsCurdir = os.getcwd() # Absolute path name of current directory
@@ -383,3 +385,5 @@ def raise_interpreter(use_locals = None):
if use_locals: local_dict = locals()
else: local_dict = globals()
code.InteractiveConsole(local_dict).interact()
+
+
diff --git a/rdiff-backup/testing/longnametest.py b/rdiff-backup/testing/longnametest.py
new file mode 100644
index 0000000..87d30bf
--- /dev/null
+++ b/rdiff-backup/testing/longnametest.py
@@ -0,0 +1,141 @@
+import unittest, errno
+from commontest import *
+from rdiff_backup import rpath, longname, Globals, regress
+
+max_len = 255
+
+class LongNameTest(unittest.TestCase):
+ """Test the longname module"""
+ root_rp = rpath.RPath(Globals.local_connection, "testfiles")
+ out_rp = root_rp.append_path('output')
+
+ def test_length_limit(self):
+ """Confirm that length limit is max_len
+
+ Some of these tests depend on the length being at most
+ max_len, so check to make sure it's accurate.
+
+ """
+ Myrm(self.out_rp.path)
+ self.out_rp.mkdir()
+
+ really_long = self.out_rp.append('a'*max_len)
+ really_long.touch()
+
+ try: too_long = self.out_rp.append("a"*(max_len+1))
+ except EnvironmentError, e:
+ assert errno.errorcode[e[0]] == 'ENAMETOOLONG', e
+ else: assert 0, "File made successfully with length " + str(max_len+1)
+
+ def make_input_dirs(self):
+ """Create two input directories with long filename(s) in them"""
+ dir1 = self.root_rp.append('longname1')
+ dir2 = self.root_rp.append('longname2')
+ Myrm(dir1.path)
+ Myrm(dir2.path)
+
+ dir1.mkdir()
+ rp11 = dir1.append('A'*max_len)
+ rp11.write_string('foobar')
+ rp12 = dir1.append('B'*max_len)
+ rp12.mkdir()
+ rp121 = rp12.append('C'*max_len)
+ rp121.touch()
+
+ dir2.mkdir()
+ rp21 = dir2.append('A'*max_len)
+ rp21.write_string('Hello, world')
+ rp22 = dir2.append('D'*max_len)
+ rp22.mkdir()
+ rp221 = rp22.append('C'*max_len)
+ rp221.touch()
+
+ return dir1, dir2
+
+ def check_dir1(self, dirrp):
+ """Make sure dirrp looks like dir1"""
+ rp1 = dirrp.append('A'*max_len)
+ assert rp1.get_data() == 'foobar', "data doesn't match"
+ rp2 = dirrp.append('B'*max_len)
+ assert rp2.isdir(), rp2
+ rp21 = rp2.append('C'*max_len)
+ assert rp21.isreg(), rp21
+
+ def check_dir2(self, dirrp):
+ """Make sure dirrp looks like dir2"""
+ rp1 = dirrp.append('A'*max_len)
+ assert rp1.get_data() == 'Hello, world', "data doesn't match"
+ rp2 = dirrp.append('D'*max_len)
+ assert rp2.isdir(), rp2
+ rp21 = rp2.append('C'*max_len)
+ assert rp21.isreg(), rp21
+
+ def generic_test(self, inlocal, outlocal, extra_args, compare_back):
+ """Used for some of the tests below"""
+ in1, in2 = self.make_input_dirs()
+ Myrm(self.out_rp.path)
+ restore_dir = self.root_rp.append('longname_out')
+
+ # Test backing up
+ rdiff_backup(inlocal, outlocal, in1.path, self.out_rp.path, 10000,
+ extra_options = extra_args)
+ if compare_back: self.check_dir1(self.out_rp)
+ rdiff_backup(inlocal, outlocal, in2.path, self.out_rp.path, 20000,
+ extra_options = extra_args)
+ if compare_back: self.check_dir2(self.out_rp)
+
+ # Now try restoring
+ Myrm(restore_dir.path)
+ rdiff_backup(inlocal, outlocal, self.out_rp.path, restore_dir.path,
+ 30000, extra_options = "-r now " + extra_args)
+ self.check_dir2(restore_dir)
+ Myrm(restore_dir.path)
+ rdiff_backup(1, 1, self.out_rp.path, restore_dir.path, 30000,
+ extra_options = "-r 10000 " + extra_args)
+ self.check_dir1(restore_dir)
+
+ def test_basic_local(self):
+ """Test backup session when increment would be too long"""
+ self.generic_test(1, 1, "", 1)
+
+ def test_quoting_local(self):
+ """Test backup session with quoting, so reg files also too long"""
+ self.generic_test(1, 1, "--override-chars-to-quote A-Z", 0)
+
+ def generic_regress_test(self, extra_args):
+ """Used for regress tests below"""
+ in1, in2 = self.make_input_dirs()
+ Myrm(self.out_rp.path)
+ restore_dir = self.root_rp.append('longname_out')
+ Myrm(restore_dir.path)
+
+ rdiff_backup(1, 1, in1.path, self.out_rp.path, 10000,
+ extra_options = extra_args)
+ rdiff_backup(1, 1, in2.path, self.out_rp.path, 20000,
+ extra_options = extra_args)
+
+ # Regress repository back to in1 condition
+ Globals.rbdir = self.out_rp.append_path('rdiff-backup-data')
+ self.add_current_mirror(10000)
+ self.out_rp.setdata()
+ regress.Regress(self.out_rp)
+
+ # Restore in1 and compare
+ rdiff_backup(1, 1, self.out_rp.path, restore_dir.path, 30000,
+ extra_options = '-r now ' + extra_args)
+ self.check_dir1(restore_dir)
+
+ def add_current_mirror(self, time):
+ """Add current_mirror marker at given time"""
+ cur_mirror_rp = Globals.rbdir.append(
+ "current_mirror.%s.data" % (Time.timetostring(time),))
+ cur_mirror_rp.touch()
+
+ def test_regress_basic(self):
+ """Test regressing when increments would be too long"""
+ self.generic_regress_test('')
+
+
+if __name__ == "__main__": unittest.main()
+
+