From 3f6645f4282082c2e140b52780a9a500c8212f82 Mon Sep 17 00:00:00 2001 From: bescoto Date: Fri, 18 Jul 2003 21:31:14 +0000 Subject: Added Daniel Hazelbaker's resource fork code, plus detection to fs_abilities and a new unittest file. git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@345 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/CHANGELOG | 5 +- rdiff-backup/rdiff_backup/Globals.py | 12 +++- rdiff-backup/rdiff_backup/Main.py | 53 ++++++++------- rdiff-backup/rdiff_backup/fs_abilities.py | 109 ++++++++++++++++++++++++------ rdiff-backup/rdiff_backup/metadata.py | 12 +++- rdiff-backup/rdiff_backup/rpath.py | 41 +++++++++-- rdiff-backup/rdiff_backup/selection.py | 20 +++--- rdiff-backup/testing/fs_abilitiestest.py | 6 +- rdiff-backup/testing/resourceforktest.py | 53 +++++++++++++++ 9 files changed, 244 insertions(+), 67 deletions(-) create mode 100644 rdiff-backup/testing/resourceforktest.py diff --git a/rdiff-backup/CHANGELOG b/rdiff-backup/CHANGELOG index aeaa67e..f907cfa 100644 --- a/rdiff-backup/CHANGELOG +++ b/rdiff-backup/CHANGELOG @@ -1,4 +1,4 @@ -New in v0.13.0 (2003/07/02) +New in v0.13.0 (2003/07/22) --------------------------- To prevent the buildup of confusing and error-prone options, the @@ -17,6 +17,9 @@ Support for access control lists (ACLs) was also added. An ACL capable file system and the python package pylibacl (which exports the posix1e module) are required. +Thanks to patches by Daniel Hazelbaker, rdiff-backup now reads and +writes Mac OS X style resource forks! + Added --list-increment-sizes switch, which tells you how much space the various backup files take up. (Suggested by Andrew Bressen) diff --git a/rdiff-backup/rdiff_backup/Globals.py b/rdiff-backup/rdiff_backup/Globals.py index 78176f2..018d016 100644 --- a/rdiff-backup/rdiff_backup/Globals.py +++ b/rdiff-backup/rdiff_backup/Globals.py @@ -71,16 +71,24 @@ read_eas = None # If true, preserve the extended attributes on the mirror directory # when backing up, or write them to the restore directory. This -# implies read_eas. +# requires read_eas. write_eas = None # If true, save access control lists when backup up. read_acls = None # If true, write access control list information to the destination -# when backing up or restoring. Implies read_acls. +# when backing up or restoring. Requires read_acls. write_acls = None +# If true, look for and save resource fork information when backing +# up. +read_resource_forks = None + +# If true, write resource fork information to destination when backing +# up or restoring. Requires read_resource_forks. +write_resource_forks = None + # This will be set as soon as the LocalConnection class loads local_connection = None diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py index 6e2af52..b3c09dd 100644 --- a/rdiff-backup/rdiff_backup/Main.py +++ b/rdiff-backup/rdiff_backup/Main.py @@ -323,29 +323,28 @@ def backup_get_mirrortime(): def backup_set_fs_globals(rpin, rpout): """Use fs_abilities to set the globals that depend on filesystem""" + def update_bool_global(attr, bool): + """If bool is not None, update Globals.attr accordingly""" + if Globals.get(attr) is not None: + SetConnections.UpdateGlobal(attr, bool) + src_fsa = fs_abilities.FSAbilities('source').init_readonly(rpin) Log(str(src_fsa), 3) - if Globals.read_acls is None: - SetConnections.UpdateGlobal('read_acls', src_fsa.acls) - if src_fsa.eas: rpin.get_ea() - if Globals.read_eas is None: - SetConnections.UpdateGlobal('read_eas', src_fsa.eas) - + update_bool_global('read_acls', src_fsa.acls) + update_bool_global('read_eas', src_fsa.eas) + update_bool_global('read_resource_forks', src_fsa.resource_forks) + dest_fsa = fs_abilities.FSAbilities('destination').init_readwrite( Globals.rbdir, override_chars_to_quote = Globals.chars_to_quote) Log(str(dest_fsa), 3) SetConnections.UpdateGlobal('preserve_hardlinks', dest_fsa.hardlinks) SetConnections.UpdateGlobal('fsync_directories', dest_fsa.fsync_dirs) - if Globals.write_acls is None: - SetConnections.UpdateGlobal('write_acls', - Globals.read_acls and dest_fsa.acls) - if Globals.write_eas is None: - SetConnections.UpdateGlobal('write_eas', - Globals.read_eas and dest_fsa.eas) SetConnections.UpdateGlobal('change_ownership', dest_fsa.ownership) - if Globals.change_dir_inc_perms is None: - SetConnections.UpdateGlobal('change_dir_inc_perms', - dest_fsa.dir_inc_perms) + update_bool_global('write_acls', Globals.read_acls and dest_fsa.acls) + update_bool_global('write_eas', Globals.read_eas and dest_fsa.eas) + update_bool_global('write_resource_forks', + Globals.read_resource_forks and dest_fsa.resource_forks) + update_bool_global('change_dir_inc_perms', dest_fsa.dir_inc_perms) SetConnections.UpdateGlobal('chars_to_quote', dest_fsa.chars_to_quote) if Globals.chars_to_quote: for conn in Globals.connections: @@ -417,20 +416,22 @@ def restore_init_quoting(src_rp): def restore_set_fs_globals(target): """Use fs_abilities to set the globals that depend on filesystem""" - target_fsa = fs_abilities.FSAbilities().init_readwrite(target, 0) - if Globals.read_acls is None: - SetConnections.UpdateGlobal('read_acls', target_fsa.acls) - if Globals.write_acls is None: - SetConnections.UpdateGlobal('write_acls', target_fsa.acls) - if Globals.read_eas is None: - SetConnections.UpdateGlobal('read_eas', target_fsa.eas) - if Globals.write_eas is None: - SetConnections.UpdateGlobal('write_eas', target_fsa.eas) - if Globals.read_eas: target.get_ea() + def update_bool_global(attr, bool): + """If bool is not None, update Globals.attr accordingly""" + if Globals.get(attr) is not None: + SetConnections.UpdateGlobal(attr, bool) + + target_fsa = fs_abilities.FSAbilities('destination').init_readwrite( + target, 0) + update_bool_global('read_acls', target_fsa.acls) + update_bool_global('write_acls', target_fsa.acls) + update_bool_global('read_eas', target_fsa.eas) + update_bool_global('write_eas', target_fsa.eas) SetConnections.UpdateGlobal('preserve_hardlinks', target_fsa.hardlinks) SetConnections.UpdateGlobal('change_ownership', target_fsa.ownership) - mirror_fsa = fs_abilities.FSAbilities().init_readwrite(Globals.rbdir) + mirror_fsa = fs_abilities.FSAbilities('source').init_readwrite( + Globals.rbdir) if Globals.chars_to_quote is None: # otherwise already overridden if mirror_fsa.chars_to_quote: SetConnections.UpdateGlobal('chars_to_quote', diff --git a/rdiff-backup/rdiff_backup/fs_abilities.py b/rdiff-backup/rdiff_backup/fs_abilities.py index c4dc509..5149c4d 100644 --- a/rdiff-backup/rdiff_backup/fs_abilities.py +++ b/rdiff-backup/rdiff_backup/fs_abilities.py @@ -28,7 +28,7 @@ FSAbilities object describing it. """ import errno -import Globals, log, TempFile +import Globals, log, TempFile, selection class FSAbilities: """Store capabilities of given file system""" @@ -38,9 +38,10 @@ class FSAbilities: eas = None # True if extended attributes supported hardlinks = None # True if hard linking supported fsync_dirs = None # True if directories can be fsync'd - read_only = None # True if capabilities were determined non-destructively dir_inc_perms = None # True if regular files can have full permissions + resource_forks = None # True if regular_file/rsrc holds resource fork name = None # Short string, not used for any technical purpose + read_only = None # True if capabilities were determined non-destructively def __init__(self, name = None): """FSAbilities initializer. name is only used in logging""" @@ -48,31 +49,50 @@ class FSAbilities: def __str__(self): """Return pretty printable version of self""" - s = ['-' * 60] + assert self.read_only == 0 or self.read_only == 1, self.read_only + s = ['-' * 65] + def addline(desc, val_text): """Add description line to s""" s.append(' %s%s%s' % (desc, ' ' * (45-len(desc)), val_text)) - if self.name: - s.append('Detected abilities for %s file system:' % (self.name,)) - else: s.append('Detected abilities for file system') - - ctq_str = (self.chars_to_quote is None and 'N/A' - or repr(self.chars_to_quote)) - addline('Characters needing quoting', ctq_str) - - for desc, val in [('Ownership changing', self.ownership), - ('Access control lists', self.acls), + def add_boolean_list(pair_list): + """Add lines from list of (desc, boolean) pairs""" + for desc, boolean in pair_list: + if boolean: val_text = 'On' + elif boolean is None: val_text = 'N/A' + else: + assert boolean == 0 + val_text = 'Off' + addline(desc, val_text) + + def get_title_line(): + """Add the first line, mostly for decoration""" + read_string = self.read_only and "read only" or "read/write" + if self.name: + return ('Detected abilities for %s (%s) file system:' % + (self.name, read_string)) + else: return ('Detected abilities for %s file system' % + (read_string,)) + + def add_ctq_line(): + """Get line describing chars to quote""" + ctq_str = (self.chars_to_quote is None and 'N/A' + or repr(self.chars_to_quote)) + addline('Characters needing quoting', ctq_str) + + s.append(get_title_line()) + if not self.read_only: + add_ctq_line() + add_boolean_list([('Ownership changing', self.ownership), + ('Hard linking', self.hardlinks), + ('fsync() directories', self.fsync_dirs), + ('Directory inc permissions', + self.dir_inc_perms)]) + add_boolean_list([('Access control lists', self.acls), ('Extended attributes', self.eas), - ('Hard linking', self.hardlinks), - ('fsync() directories', self.fsync_dirs), - ('Directory inc permissions', self.dir_inc_perms)]: - if val: val_text = 'On' - elif val is None: val_text = 'N/A' - else: - assert val == 0 - val_text = 'Off' - addline(desc, val_text) + ('Mac OS X style resource forks', + self.resource_forks)]) s.append(s[0]) return '\n'.join(s) @@ -90,6 +110,7 @@ class FSAbilities: self.read_only = 1 self.set_eas(rp, 0) self.set_acls(rp) + self.set_resource_fork_readonly(rp) return self def init_readwrite(self, rbdir, use_ctq_file = 1, @@ -121,6 +142,7 @@ class FSAbilities: self.set_eas(subdir, 1) self.set_acls(subdir) self.set_dir_inc_perms(subdir) + self.set_resource_fork_readwrite(subdir) if override_chars_to_quote is None: self.set_chars_to_quote(subdir) else: self.chars_to_quote = override_chars_to_quote if use_ctq_file: self.compare_chars_to_quote(rbdir) @@ -258,6 +280,49 @@ rdiff-backup-data/chars_to_quote. else: self.dir_inc_perms = 0 test_rp.delete() + def set_resource_fork_readwrite(self, dir_rp): + """Test for resource forks by writing to regular_file/rsrc""" + reg_rp = dir_rp.append('regfile') + reg_rp.touch() + rfork = reg_rp.append('rsrc') + assert not rfork.lstat() + + s = 'test string---this should end up in resource fork' + try: + fp_write = rfork.open('wb') + fp_write.write(s) + assert not fp_write.close() + + fp_read = rfork.open('rb') + s_back = fp_read.read() + assert not fp.read.close() + except (OSError, IOError), e: self.resource_forks = 0 + else: self.resource_forks = (s_back == s) + reg_rp.delete() + + def set_resource_fork_readonly(self, dir_rp): + """Test for resource fork support by testing an regular file + + Launches search for regular file in given directory. If no + regular file is found, resource_fork support will be turned + off by default. + + """ + for rp in selection.Select(dir_rp).set_iter(): + if rp.isreg(): + try: + rfork = rp.append('rsrc') + fp = rfork.open('rb') + fp.read() + assert not fp.close() + except (OSError, IOError), e: + self.resource_forks = 0 + return + self.resource_forks = 1 + return + self.resource_forks = 0 + + def test_eas_local(rp, write): """Test ea support. Must be called locally. Usedy by set_eas above.""" assert Globals.local_connection is rp.conn diff --git a/rdiff-backup/rdiff_backup/metadata.py b/rdiff-backup/rdiff_backup/metadata.py index 3d8ba60..eec8e3e 100644 --- a/rdiff-backup/rdiff_backup/metadata.py +++ b/rdiff-backup/rdiff_backup/metadata.py @@ -55,7 +55,7 @@ field names and values. """ from __future__ import generators -import re, gzip, os +import re, gzip, os, binascii import log, Globals, rpath, Time, robust, increment, static class ParsingError(Exception): @@ -74,6 +74,12 @@ def RORP2Record(rorpath): if type == "reg": str_list.append(" Size %s\n" % rorpath.getsize()) + # If there is a resource fork, save it. + if rorpath.has_resource_fork(): + if not rorpath.get_resource_fork(): rf = "None" + else: rf = binascii.hexlify(rorpath.get_resource_fork()) + str_list.append(" ResourceFork %s\n" % (rf,)) + # If file is hardlinked, add that information if Globals.preserve_hardlinks: numlinks = rorpath.getnumlinks() @@ -81,6 +87,7 @@ def RORP2Record(rorpath): str_list.append(" NumHardLinks %s\n" % numlinks) str_list.append(" Inode %s\n" % rorpath.getinode()) str_list.append(" DeviceLoc %s\n" % rorpath.getdevloc()) + elif type == "None": return "".join(str_list) elif type == "dir" or type == "sock" or type == "fifo": pass elif type == "sym": @@ -122,6 +129,9 @@ def Record2RORP(record_string): if data == "None": data_dict['type'] = None else: data_dict['type'] = data elif field == "Size": data_dict['size'] = long(data) + elif field == "ResourceFork": + if data == "None": data_dict['resourcefork'] = "" + else: data_dict['resourcefork'] = binascii.unhexlify(data) elif field == "NumHardLinks": data_dict['nlink'] = int(data) elif field == "Inode": data_dict['inode'] = long(data) elif field == "DeviceLoc": data_dict['devloc'] = long(data) diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py index efad6ed..8308b87 100644 --- a/rdiff-backup/rdiff_backup/rpath.py +++ b/rdiff-backup/rdiff_backup/rpath.py @@ -157,6 +157,8 @@ def copy_attribs(rpin, rpout, acls = 1): if Globals.change_ownership: apply(rpout.chown, rpin.getuidgid()) if Globals.change_permissions: rpout.chmod(rpin.getperms()) if Globals.write_acls and acls: rpout.write_acl(rpin.get_acl()) + if Globals.write_resource_forks and rpin.isreg() and rpout.isreg(): + rpout.write_resource_fork(rpin.get_resource_fork()) if not rpin.isdev(): rpout.setmtime(rpin.getmtime()) def cmp_attribs(rp1, rp2): @@ -271,6 +273,8 @@ class RORPath: elif key == 'size' and not self.isreg(): pass elif key == 'ea' and not Globals.read_eas: pass elif key == 'acl' and not Globals.read_acls: pass + elif key == 'resourcefork' and not Globals.read_resource_forks: + pass elif (key == 'inode' and (not self.isreg() or self.getnumlinks() == 1 or not Globals.compare_inode or @@ -303,10 +307,10 @@ class RORPath: elif key == 'size' and not self.isreg(): pass elif key == 'perms' and not Globals.change_permissions: pass elif key == 'inode': pass - elif (key == 'ea' and - not (Globals.read_eas and Globals.write_eas)): pass - elif (key == 'acl' and - not (Globals.read_acls and Globals.write_acls)): pass + elif key == 'ea' and not Globals.write_eas: pass + elif key == 'acl' and not Globals.write_acls: pass + elif key == 'resourcefork' and not Globals.write_resource_forks: + pass elif (not other.data.has_key(key) or self.data[key] != other.data[key]): return 0 return 1 @@ -547,6 +551,18 @@ class RORPath: """Return extended attributes object""" return self.data['ea'] + def has_resource_fork(self): + """True if rpath has a resourcefork parameter""" + return self.data.has_key('resourcefork') + + def get_resource_fork(self): + """Return the resource fork in binary data""" + return self.data['resourcefork'] + + def set_resource_fork(self, rfork): + """Record resource fork in dictionary. Does not write""" + self.data['resourcefork'] = rfork + class RPath(RORPath): """Remote Path class - wrapper around a possibly non-local pathname @@ -608,6 +624,8 @@ class RPath(RORPath): self.data = self.conn.C.make_file_dict(self.path) if Globals.read_eas and self.lstat(): self.get_ea() if Globals.read_acls and self.lstat(): self.get_acl() + if Globals.read_resource_forks and self.isreg(): + self.get_resource_fork() def make_file_dict_old(self): """Create the data dictionary""" @@ -998,6 +1016,21 @@ class RPath(RORPath): ea.write_to_rp(self) self.data['ea'] = ea + def get_resource_fork(self): + """Return resource fork data, setting if necessary""" + assert self.isreg() + try: rfork = self.data['resourcefork'] + except KeyError: + rfork = self.append('rsrc').get_data() + self.data['resourcefork'] = rfork + return rfork + + def write_resource_fork(self, rfork_data): + """Write new resource fork to self""" + fp = self.append('rsrc').open('wb') + fp.write(rfork_data) + assert not fp.close() + class RPathFileHook: """Look like a file, but add closing hook""" diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py index 29b12e8..e87ef8b 100644 --- a/rdiff-backup/rdiff_backup/selection.py +++ b/rdiff-backup/rdiff_backup/selection.py @@ -144,7 +144,7 @@ class Select: delayed_rp_stack.append(rpath) diryield_stack.append(diryield(rpath)) - def Iterate(self, rpath, rec_func, sel_func): + def Iterate(self, rp, rec_func, sel_func): """Return iterator yielding rpaths in rpath rec_func is usually the same as this function and is what @@ -155,21 +155,21 @@ class Select: is usually self.Select. """ - s = sel_func(rpath) + s = sel_func(rp) if s == 0: return elif s == 1: # File is included - yield rpath - if rpath.isdir(): - for rp in self.iterate_in_dir(rpath, rec_func, sel_func): - yield rp + yield rp + if rp.isdir(): + for rp2 in self.iterate_in_dir(rp, rec_func, sel_func): + yield rp2 elif s == 2: - if rpath.isdir(): # Directory is merely scanned - iid = self.iterate_in_dir(rpath, rec_func, sel_func) + if rp.isdir(): # Directory is merely scanned + iid = self.iterate_in_dir(rp, rec_func, sel_func) try: first = iid.next() except StopIteration: return # no files inside; skip rp - yield rpath + yield rp yield first - for rp in iid: yield rp + for rp2 in iid: yield rp2 else: assert 0, "Invalid selection result %s" % (str(s),) def listdir(self, dir_rp): diff --git a/rdiff-backup/testing/fs_abilitiestest.py b/rdiff-backup/testing/fs_abilitiestest.py index c16ab46..cdbe75e 100644 --- a/rdiff-backup/testing/fs_abilitiestest.py +++ b/rdiff-backup/testing/fs_abilitiestest.py @@ -18,6 +18,7 @@ class FSAbilitiesTest(unittest.TestCase): ownership = (os.getuid() == 0) hardlinks = fsync_dirs = 1 dir_inc_perms = 1 + resource_forks = 0 # Describes MS-Windows style file system #dir_to_test = "/mnt/fat" @@ -25,7 +26,8 @@ class FSAbilitiesTest(unittest.TestCase): #chars_to_quote = "^a-z0-9_ -" #ownership = hardlinks = 0 #fsync_dirs = 1 - #dir_inc_perms = XXX + #dir_inc_perms = 0 + #resource_forks = 0 def testReadOnly(self): """Test basic querying read only""" @@ -35,6 +37,7 @@ class FSAbilitiesTest(unittest.TestCase): assert fsa.read_only == 1, fsa.read_only assert fsa.eas == self.eas, fsa.eas assert fsa.acls == self.acls, fsa.acls + assert fsa.resource_forks == self.resource_forks, fsa.resource_forks def testReadWrite(self): """Test basic querying read/write""" @@ -55,6 +58,7 @@ class FSAbilitiesTest(unittest.TestCase): assert fsa.hardlinks == self.hardlinks, fsa.hardlinks assert fsa.fsync_dirs == self.fsync_dirs, fsa.fsync_dirs assert fsa.dir_inc_perms == self.dir_inc_perms, fsa.dir_inc_perms + assert fsa.resource_forks == self.resource_forks, fsa.resource_forks ctq_rp = new_dir.append("chars_to_quote") assert ctq_rp.lstat() diff --git a/rdiff-backup/testing/resourceforktest.py b/rdiff-backup/testing/resourceforktest.py new file mode 100644 index 0000000..09cc1d0 --- /dev/null +++ b/rdiff-backup/testing/resourceforktest.py @@ -0,0 +1,53 @@ +import unittest +from commontest import * +from rdiff_backup import rpath +from rdiff_backup import metadata + +"""***NOTE*** + +None of these tests should work unless your system supports resource +forks. So basically these tests should only be run on Mac OS X. + +""" + +Globals.read_resource_forks = Globals.write_resource_forks = 1 + +class ResourceForkTest(unittest.TestCase): + """Test dealing with Mac OS X style resource forks""" + tempdir = rpath.RPath(Globals.local_connection, + 'testfiles/resource_fork_test') + def make_temp(self): + """Make temp directory testfiles/resource_fork_test""" + if self.tempdir.lstat(): self.tempdir.delete() + self.tempdir.mkdir() + + def testBasic(self): + """Test basic reading and writing of resource forks""" + self.make_temp() + rp = self.tempdir.append('test') + rp.touch() + assert rp.get_resource_fork() == '', rp.get_resource_fork() + + s = 'new resource fork data' + rp.write_resource_fork(s) + assert rp.get_resource_fork() == s, rp.get_resource_fork() + + rp2 = self.tempdir.append('test') + assert rp2.isreg() + assert rp2.get_resource_fork() == s, rp2.get_resource_fork() + + def testRecord(self): + """Test reading, writing, and comparing of records with rforks""" + self.make_temp() + rp = self.tempdir.append('test') + rp.touch() + rp.set_resource_fork('hello') + + record = metadata.RORP2Record(rp) + #print record + rorp_out = metadata.Record2RORP(record) + assert rorp_out == rp, (rorp_out, rp) + assert rorp_out.get_resource_fork() == 'hello' + + +if __name__ == "__main__": unittest.main() -- cgit v1.2.1