added "wrappers" sub-package, and a SimulateXAttrs wrapper

git-svn-id: http://pyfilesystem.googlecode.com/svn/branches/rfk-ideas@139 67cdc799-7952-0410-af00-57a81ceafa0f

6 files changed, 377 insertions, 126 deletions

diff --git a/fs/base.py b/fs/base.py
index 5c0e41d..33def75 100644
--- a/fs/base.py
+++ b/fs/base.py
@@ -537,15 +537,12 @@ class FS(object):
                      will be raised.
         chunk_size -- Size of chunks to use if a simple copy is required
 
-        If the destination path exists and is a directory, the file will
-        be copied into that directory.
         """
 
-        if self.isdir(dst):
-            dst = pathjoin( dirname(dst), resourcename(src) )
-
         if not self.isfile(src):
-            raise ResourceInvalidError(src,msg="Source is not a file: %(path)s")
+            if self.isdir(src):
+                raise ResourceInvalidError(src,msg="Source is not a file: %(path)s")
+            raise FileNotFoundError(src)
         if not overwrite and self.exists(dst):
             raise DestinationExistsError(dst)
 
@@ -588,7 +585,9 @@ class FS(object):
 
         if src_syspath is not None and dst_syspath is not None:
             if not self.isfile(src):
-                raise ResourceInvalidError(src, msg="Source is not a file: %(path)s")
+                if self.isdir(src):
+                    raise ResourceInvalidError(src,msg="Source is not a file: %(path)s")
+                raise FileNotFoundError(src)
             if not overwrite and self.exists(dst):
                 raise DestinationExistsError(dst)
             shutil.move(src_syspath, dst_syspath)
@@ -597,48 +596,6 @@ class FS(object):
             self.remove(src)
 
 
-#    def _get_attr_path(self, path):
-#        if self.isdir(path):
-#            return pathjoin(path, '.xattrs.')
-#        else:
-#            dir_path, file_path = pathsplit(path)
-#            return pathjoin(dir_path, '.xattrs.'+file_path)
-#
-#    def _get_attr_dict(self, path):
-#        attr_path = self._get_attr_path(path)
-#        if self.exists(attr_path):
-#            return pickle.loads(self.getcontents(attr_path))
-#        else:
-#            return {}
-#
-#    def _set_attr_dict(self, path, attrs):
-#        attr_path = self._get_attr_path(path)
-#        self.setcontents(self._get_attr_path(path), pickle.dumps(attrs))
-#
-#    def setxattr(self, path, key, value):
-#        attrs = self._get_attr_dict(path)
-#        attrs[key] = value
-#        self._set_attr_dict(path, attrs)
-#
-#    def getxattr(self, path, key, default):
-#        attrs = self._get_attr_dict(path)
-#        return attrs.get(key, default)
-#
-#    def delxattr(self, path, key):
-#        attrs = self._get_attr_dict(path)
-#        try:
-#            del attrs[key]
-#        except KeyError:
-#            pass
-#        self._set_attr_dict(path, attrs)
-#
-#    def listxattrs(self, path):
-#        attrs = self._get_attr_dict(path)
-#        return self._get_attr_dict(path).keys()
-#
-#    def getxattrs(self, path):
-#        return dict( [(k, self.getxattr(path, k)) for k in self.listxattrs(path)] )
-
     def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
         """Moves a directory from one location to another.
 
diff --git a/fs/errors.py b/fs/errors.py
index b216c64..e3eb618 100644
--- a/fs/errors.py
+++ b/fs/errors.py
@@ -30,7 +30,7 @@ class PathError(FSError):
 
     def __init__(self,path,**kwds):
         self.path = path
-        super(ResourceError,self).__init__(**kwds)
+        super(PathError,self).__init__(**kwds)
  
 
 class OperationFailedError(FSError):
diff --git a/fs/osfs.py b/fs/osfs.py
index 7b675cb..0894765 100644
--- a/fs/osfs.py
+++ b/fs/osfs.py
@@ -166,66 +166,31 @@ class OSFS(FS):
         return stats.st_size
 
 
-#    def setxattr(self, path, key, value):
-#        self._lock.acquire()
-#        try:
-#            if xattr is None:
-#                return FS.setxattr(self, path, key, value)
-#            try:
-#                xattr.xattr(self.getsyspath(path))[key]=value
-#            except IOError, e:
-#                if e.errno == 95:
-#                    return FS.setxattr(self, path, key, value)
-#                else:
-#                    raise OperationFailedError('XATTR_FAILED', path, details=e)
-#        finally:
-#            self._lock.release()
-#
-#    def getxattr(self, path, key, default=None):
-#        self._lock.acquire()
-#        try:
-#            if xattr is None:
-#                return FS.getxattr(self, path, key, default)
-#            try:
-#                return xattr.xattr(self.getsyspath(path)).get(key)
-#            except IOError, e:
-#                if e.errno == 95:
-#                    return FS.getxattr(self, path, key, default)
-#                else:
-#                    raise OperationFailedError('XATTR_FAILED', path, details=e)
-#        finally:
-#            self._lock.release()
-#
-#    def removexattr(self, path, key):
-#        self._lock.acquire()
-#        try:
-#            if xattr is None:
-#                return FS.removexattr(self, path, key)
-#            try:
-#                del xattr.xattr(self.getsyspath(path))[key]
-#            except KeyError:
-#                pass
-#            except IOError, e:
-#                if e.errono == 95:
-#                    return FS.removexattr(self, path, key)
-#                else:
-#                    raise OperationFailedError('XATTR_FAILED', path, details=e)
-#        finally:
-#            self._lock.release()
-#
-#    def listxattrs(self, path):
-#        self._lock.acquire()
-#        try:
-#            if xattr is None:
-#                return FS.listxattrs(self, path)
-#            try:
-#                return xattr.xattr(self.getsyspath(path)).keys()
-#            except IOError, e:
-#                if errono == 95:
-#                    return FS.listxattrs(self, path)
-#                else:
-#                    raise OperationFailedError('XATTR_FAILED', path, details=e)
-#        finally:
-#            self._lock.release()
-#
+    #  Provide native xattr support if available
+    if xattr:
+        def setxattr(self, path, key, value):
+            try:
+                xattr.xattr(self.getsyspath(path))[key]=value
+            except IOError, e:
+                raise OperationFailedError('set extended attribute', path=path, details=e)
+
+        def getxattr(self, path, key, default=None):
+            try:
+                return xattr.xattr(self.getsyspath(path)).get(key,default)
+            except IOError, e:
+                raise OperationFailedError('get extended attribute', path=path, details=e)
+
+        def delxattr(self, path, key):
+            try:
+                del xattr.xattr(self.getsyspath(path))[key]
+            except KeyError:
+                pass
+            except IOError, e:
+                raise OperationFailedError('delete extended attribute', path=path, details=e)
+
+        def xattrs(self, path):
+            try:
+                return xattr.xattr(self.getsyspath(path)).keys()
+            except IOError, e:
+                raise OperationFailedError('list extended attributes', path=path, details=e)
 
diff --git a/fs/tests.py b/fs/tests.py
index 9fd9a26..0f2a0d9 100644
--- a/fs/tests.py
+++ b/fs/tests.py
@@ -18,21 +18,19 @@ from helpers import *
 ####################
 
 
-class BaseFSTestCases:
+class FSTestCases:
     """Base suite of testcases for filesystem implementations.
 
     Any FS subclass should be capable of passing all of these tests.
     To apply the tests to your own FS implementation, simply subclass
-    BaseFSTestCase and have the setUp method set self.fs to an instance
+    FSTestCase and have the setUp method set self.fs to an instance
     of your FS implementation.
 
-    Note that you'll also need to subclass
+    Note that you'll also need to subclass unittest.TestCase - this class
+    is designed as a mixin so that it's not picked up by test tools such
+    as nose.
     """
 
-    def setUp(self):
-        import nose
-        raise nose.SkipTest("this is an abstract base class")
-        
     def check(self, p):
         """Check that a file exists within self.fs"""
         return self.fs.exists(p)
@@ -443,6 +441,76 @@ class BaseFSTestCases:
 
 
 
+class XAttrTestCases:
+    """Testcases for filesystems providing extended attribute support."""
+
+    def test_getsetdel(self):
+        def do_getsetdel(p):
+            self.assertEqual(self.fs.getxattr(p,"xattr1"),None)
+            self.fs.setxattr(p,"xattr1","value1")
+            self.assertEqual(self.fs.getxattr(p,"xattr1"),"value1")
+            self.fs.delxattr(p,"xattr1")
+            self.assertEqual(self.fs.getxattr(p,"xattr1"),None)
+        self.fs.createfile("test.txt","hello")
+        do_getsetdel("test.txt")
+        self.assertRaises(fs.ResourceNotFoundError,self.fs.getxattr,"test2.txt","xattr1")
+        self.fs.makedir("mystuff")
+        self.fs.createfile("/mystuff/test.txt","")
+        do_getsetdel("mystuff")
+        do_getsetdel("mystuff/test.txt")
+
+    def test_list_xattrs(self):
+        def do_list(p):
+            self.assertEquals(self.fs.xattrs(p),[])
+            self.fs.setxattr(p,"xattr1","value1")
+            self.assertEquals(self.fs.xattrs(p),["xattr1"])
+            self.fs.setxattr(p,"attr2","value2")
+            self.assertEquals(sorted(self.fs.xattrs(p)),["attr2","xattr1"])
+            self.fs.delxattr(p,"xattr1")
+            self.assertEquals(self.fs.xattrs(p),["attr2"])
+            self.fs.delxattr(p,"attr2")
+            self.assertEquals(self.fs.xattrs(p),[])
+        self.fs.createfile("test.txt","hello")
+        do_list("test.txt")
+        self.fs.makedir("mystuff")
+        self.fs.createfile("/mystuff/test.txt","")
+        do_list("mystuff")
+        do_list("mystuff/test.txt")
+
+    def test_copy_xattrs(self):
+        self.fs.createfile("a.txt","content")
+        self.fs.setxattr("a.txt","myattr","myvalue")
+        self.fs.setxattr("a.txt","testattr","testvalue")
+        self.fs.makedir("stuff")
+        self.fs.copy("a.txt","stuff/a.txt")
+        self.assertTrue(self.fs.exists("stuff/a.txt"))
+        self.assertEquals(self.fs.getxattr("stuff/a.txt","myattr"),"myvalue")
+        self.assertEquals(self.fs.getxattr("stuff/a.txt","testattr"),"testvalue")
+        self.assertEquals(self.fs.getxattr("a.txt","myattr"),"myvalue")
+        self.assertEquals(self.fs.getxattr("a.txt","testattr"),"testvalue")
+        self.fs.setxattr("stuff","dirattr","a directory")
+        self.fs.copydir("stuff","stuff2")
+        self.assertEquals(self.fs.getxattr("stuff2/a.txt","myattr"),"myvalue")
+        self.assertEquals(self.fs.getxattr("stuff2/a.txt","testattr"),"testvalue")
+        self.assertEquals(self.fs.getxattr("stuff2","dirattr"),"a directory")
+        self.assertEquals(self.fs.getxattr("stuff","dirattr"),"a directory")
+
+    def test_move_xattrs(self):
+        self.fs.createfile("a.txt","content")
+        self.fs.setxattr("a.txt","myattr","myvalue")
+        self.fs.setxattr("a.txt","testattr","testvalue")
+        self.fs.makedir("stuff")
+        self.fs.move("a.txt","stuff/a.txt")
+        self.assertTrue(self.fs.exists("stuff/a.txt"))
+        self.assertEquals(self.fs.getxattr("stuff/a.txt","myattr"),"myvalue")
+        self.assertEquals(self.fs.getxattr("stuff/a.txt","testattr"),"testvalue")
+        self.fs.setxattr("stuff","dirattr","a directory")
+        self.fs.movedir("stuff","stuff2")
+        self.assertEquals(self.fs.getxattr("stuff2/a.txt","myattr"),"myvalue")
+        self.assertEquals(self.fs.getxattr("stuff2/a.txt","testattr"),"testvalue")
+        self.assertEquals(self.fs.getxattr("stuff2","dirattr"),"a directory")
+ 
+
 ####################
 
 
@@ -577,7 +645,7 @@ class TestObjectTree(unittest.TestCase):
 import osfs
 import os
 
-class TestOSFS(unittest.TestCase,BaseFSTestCases):
+class TestOSFS(unittest.TestCase,FSTestCases):
 
     def setUp(self):
         self.temp_dir = tempfile.mkdtemp("fstest")
@@ -591,7 +659,7 @@ class TestOSFS(unittest.TestCase,BaseFSTestCases):
 
 
 
-class TestSubFS(unittest.TestCase,BaseFSTestCases):
+class TestSubFS(unittest.TestCase,FSTestCases):
 
     def setUp(self):
         self.temp_dir = tempfile.mkdtemp("fstest")
@@ -609,14 +677,14 @@ class TestSubFS(unittest.TestCase,BaseFSTestCases):
 
 
 import memoryfs
-class TestMemoryFS(unittest.TestCase,BaseFSTestCases):
+class TestMemoryFS(unittest.TestCase,FSTestCases):
 
     def setUp(self):
         self.fs = memoryfs.MemoryFS()
 
 
 import mountfs
-class TestMountFS(unittest.TestCase,BaseFSTestCases):
+class TestMountFS(unittest.TestCase,FSTestCases):
 
     def setUp(self):
         self.mount_fs = mountfs.MountFS()
@@ -632,7 +700,7 @@ class TestMountFS(unittest.TestCase,BaseFSTestCases):
 
 
 import tempfs
-class TestTempFS(unittest.TestCase,BaseFSTestCases):
+class TestTempFS(unittest.TestCase,FSTestCases):
 
     def setUp(self):
         self.fs = tempfs.TempFS()
@@ -648,7 +716,7 @@ class TestTempFS(unittest.TestCase,BaseFSTestCases):
 
 
 import s3fs
-class TestS3FS(unittest.TestCase,BaseFSTestCases):
+class TestS3FS(unittest.TestCase,FSTestCases):
 
     bucket = "test-s3fs.rfk.id.au"
 
@@ -673,7 +741,7 @@ class TestS3FS(unittest.TestCase,BaseFSTestCases):
 import rpcfs
 import socket
 import threading
-class TestRPCFS(unittest.TestCase,BaseFSTestCases):
+class TestRPCFS(unittest.TestCase,FSTestCases):
 
     def setUp(self):
         self.port = 8000
@@ -708,6 +776,22 @@ class TestRPCFS(unittest.TestCase,BaseFSTestCases):
           pass
 
 
+from fs.wrappers.xattr import SimulateXAttr
+class TestSimulateXAttr(unittest.TestCase,FSTestCases,XAttrTestCases):
+
+    def setUp(self):
+        self.fs = SimulateXAttr(tempfs.TempFS())
+
+    def tearDown(self):
+        td = self.fs._temp_dir
+        self.fs.close()
+        self.assert_(not os.path.exists(td))
+
+    def check(self, p):
+        td = self.fs._temp_dir
+        return os.path.exists(os.path.join(td, makerelative(p)))
+
+
 ####################
 
 
diff --git a/fs/wrappers/__init__.py b/fs/wrappers/__init__.py
new file mode 100644
index 0000000..4221d78
--- /dev/null
+++ b/fs/wrappers/__init__.py
@@ -0,0 +1,130 @@
+"""
+
+  fs.wrappers:  clases to transform files in a filesystem
+
+This sub-module provides facilities for easily wrapping an FS object inside
+some sort of transformation - for example, to transparently compress or encrypt
+files.
+
+"""
+
+from fs.base import FS
+
+class FSWrapper(FS):
+    """FS that wraps another FS, providing translation etc.
+
+    This class allows simple transforms to be applied to the names
+    and/or contents of files in an FS.  It's particularly handy in
+    conjunction with wrappers from the "filelike" module.
+    """
+
+    def __init__(self,fs):
+        super(FSWrapper,self).__init__()
+        self.wrapped_fs = fs
+
+    def _file_wrap(self,f,mode):
+        """Apply wrapping to an opened file."""
+        return f
+
+    def _encode_name(self,name):
+        """Encode path component for the underlying FS."""
+        return name
+
+    def _decode_name(self,name):
+        """Decode path component from the underlying FS."""
+        return name
+
+    def _encode(self,path):
+        """Encode path for the underlying FS."""
+        names = path.split("/")
+        e_names = []
+        for name in names:
+            if name == "":
+                e_names.append("")
+            else:
+                e_names.append(self._encode_name(name))
+        return "/".join(e_names)
+
+    def _decode(self,path):
+        """Decode path from the underlying FS."""
+        names = path.split("/")
+        d_names = []
+        for name in names:
+            if name == "":
+                d_names.append("")
+            else:
+                d_names.append(self._decode_name(name))
+        return "/".join(d_names)
+
+    def _adjust_mode(self,mode):
+        """Adjust the mode used to open a file in the underlying FS.
+
+        This method takes the mode given when opening a file, and should
+        return a two-tuple giving the mode to be used in this FS as first
+        item, and the mode to be used in the underlying FS as the second.
+        """
+        return (mode,mode)
+
+    def getsyspath(self,path,allow_none=False):
+        return self.wrapped_fs.getsyspath(self._encode(path),allow_none)
+
+    def hassyspath(self,path):
+        return self.wrapped_fs.hassyspath(self._encode(path))
+
+    def open(self,path,mode="r"):
+        (mode,wmode) = self._adjust_mode(mode)
+        f = self.wrapped_fs.open(self._encode(path),wmode)
+        return self._file_wrap(f,mode)
+
+    def exists(self,path):
+        return self.wrapped_fs.exists(self._encode(path))
+
+    def isdir(self,path):
+        return self.wrapped_fs.isdir(self._encode(path))
+
+    def isfile(self,path):
+        return self.wrapped_fs.isfile(self._encode(path))
+
+    def listdir(self,path="",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
+        entries = []
+        for name in self.wrapped_fs.listdir(self._encode(path),wildcard=None,full=full,absolute=absolute,dirs_only=dirs_only,files_only=files_only):
+            entries.append(self._decode(name))
+        return self._listdir_helper(path,entries,wildcard=wildcard,full=False,absolute=False,dirs_only=False,files_only=False)
+
+    def makedir(self,path,*args,**kwds):
+        return self.wrapped_fs.makedir(self._encode(path),*args,**kwds)
+
+    def remove(self,path):
+        return self.wrapped_fs.remove(self._encode(path))
+
+    def removedir(self,path,*args,**kwds):
+        return self.wrapped_fs.removedir(self._encode(path),*args,**kwds)
+
+    def rename(self,src,dst):
+        return self.wrapped_fs.rename(self._encode(src),self._encode(dst))
+
+    def getinfo(self,path):
+        return self.wrapped_fs.getinfo(self._encode(path))
+
+    def desc(self,path):
+        return self.wrapped_fs.desc(self._encode(path))
+
+    def copy(self,src,dst,overwrite=False,chunk_size=16384):
+        return self.wrapped_fs.copy(self._encode(src),self._encode(dst),overwrite,chunk_size)
+
+    def move(self,src,dst,overwrite=False,chunk_size=16384):
+        return self.wrapped_fs.move(self._encode(src),self._encode(dst),overwrite,chunk_size)
+
+    def movedir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
+        return self.wrapped_fs.movedir(self._encode(src),self._encode(dst),overwrite,ignore_errors,chunk_size)
+
+    def copydir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
+        return self.wrapped_fs.copydir(self._encode(src),self._encode(dst),overwrite,ignore_errors,chunk_size)
+
+    def __getattr__(self,attr):
+        return getattr(self.wrapped_fs,attr)
+
+    def close(self):
+        if hasattr(self.wrapped_fs,"close"):
+            self.wrapped_fs.close()
+
diff --git a/fs/wrappers/xattr.py b/fs/wrappers/xattr.py
new file mode 100644
index 0000000..3eb5820
--- /dev/null
+++ b/fs/wrappers/xattr.py
@@ -0,0 +1,115 @@
+"""
+
+  fs.wrappers.xattr:  FS wrapper for simulating extended-attribute support.
+
+"""
+
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+from fs.helpers import *
+from fs.errors import *
+from fs.wrappers import FSWrapper
+
+class SimulateXAttr(FSWrapper):
+    """FS wrapper class that simulates xattr support.
+
+    For each file in the underlying FS, this class managed a corresponding 
+    '.xattr' file containing its extended attributes.
+    """
+
+    def _get_attr_path(self, path):
+        """Get the path of the file containing xattrs for the given path."""
+        if self.wrapped_fs.isdir(path):
+            return pathjoin(path, '.xattrs.')
+        else:
+            dir_path, file_name = pathsplit(path)
+            return pathjoin(dir_path, '.xattrs.'+file_name)
+
+    def _is_attr_path(self, path):
+        """Check whether the given path references an xattrs file."""
+        _,name = pathsplit(path)
+        if name.startswith(".xattrs."):
+            return True
+        return False
+
+    def _get_attr_dict(self, path):
+        """Retrieve the xattr dictionary for the given path."""
+        attr_path = self._get_attr_path(path)
+        if self.wrapped_fs.exists(attr_path):
+            return pickle.loads(self.wrapped_fs.getcontents(attr_path))
+        else:
+            return {}
+
+    def _set_attr_dict(self, path, attrs):
+        """Store the xattr dictionary for the given path."""
+        attr_path = self._get_attr_path(path)
+        self.wrapped_fs.setcontents(self._get_attr_path(path), pickle.dumps(attrs))
+
+    def setxattr(self, path, key, value):
+        """Set an extended attribute on the given path."""
+        if not self.exists(path):
+            raise ResourceNotFoundError(path)
+        attrs = self._get_attr_dict(path)
+        attrs[key] = value
+        self._set_attr_dict(path, attrs)
+
+    def getxattr(self, path, key, default=None):
+        """Retrieve an extended attribute for the given path."""
+        if not self.exists(path):
+            raise ResourceNotFoundError(path)
+        attrs = self._get_attr_dict(path)
+        return attrs.get(key, default)
+
+    def delxattr(self, path, key):
+        if not self.exists(path):
+            raise ResourceNotFoundError(path)
+        attrs = self._get_attr_dict(path)
+        try:
+            del attrs[key]
+        except KeyError:
+            pass
+        self._set_attr_dict(path, attrs)
+
+    def xattrs(self,path):
+        """List all the extended attribute keys set on the given path."""
+        if not self.exists(path):
+            raise ResourceNotFoundError(path)
+        return self._get_attr_dict(path).keys()
+
+    def _encode(self,path):
+        """Prevent requests for operations on .xattr files."""
+        if self._is_attr_path(path):
+            raise PathError(path,msg="Paths cannot contain '.xattrs.': %(path)s")
+        return path
+
+    def _decode(self,path):
+        return path
+
+    def listdir(self,path="",**kwds):
+        """Prevent .xattr from appearing in listings."""
+        entries = self.wrapped_fs.listdir(path,**kwds)
+        return [e for e in entries if not self._is_attr_path(e)]
+
+    def copy(self,src,dst,**kwds):
+        """Ensure xattrs are copied when copying a file."""
+        self.wrapped_fs.copy(self._encode(src),self._encode(dst),**kwds)
+        s_attr_file = self._get_attr_path(src)
+        d_attr_file = self._get_attr_path(dst)
+        try:
+            self.wrapped_fs.copy(s_attr_file,d_attr_file,overwrite=True)
+        except ResourceNotFoundError,e:
+            pass
+
+    def move(self,src,dst,**kwds):
+        """Ensure xattrs are preserved when moving a file."""
+        self.wrapped_fs.move(self._encode(src),self._encode(dst),**kwds)
+        s_attr_file = self._get_attr_path(src)
+        d_attr_file = self._get_attr_path(dst)
+        try:
+            self.wrapped_fs.move(s_attr_file,d_attr_file,overwrite=True)
+        except ResourceNotFoundError:
+            pass
+