diff options
-rw-r--r-- | fs/base.py | 55 | ||||
-rw-r--r-- | fs/errors.py | 2 | ||||
-rw-r--r-- | fs/osfs.py | 89 | ||||
-rw-r--r-- | fs/tests.py | 112 | ||||
-rw-r--r-- | fs/wrappers/__init__.py | 130 | ||||
-rw-r--r-- | fs/wrappers/xattr.py | 115 |
6 files changed, 377 insertions, 126 deletions
@@ -537,15 +537,12 @@ class FS(object): will be raised. chunk_size -- Size of chunks to use if a simple copy is required - If the destination path exists and is a directory, the file will - be copied into that directory. """ - if self.isdir(dst): - dst = pathjoin( dirname(dst), resourcename(src) ) - if not self.isfile(src): - raise ResourceInvalidError(src,msg="Source is not a file: %(path)s") + if self.isdir(src): + raise ResourceInvalidError(src,msg="Source is not a file: %(path)s") + raise FileNotFoundError(src) if not overwrite and self.exists(dst): raise DestinationExistsError(dst) @@ -588,7 +585,9 @@ class FS(object): if src_syspath is not None and dst_syspath is not None: if not self.isfile(src): - raise ResourceInvalidError(src, msg="Source is not a file: %(path)s") + if self.isdir(src): + raise ResourceInvalidError(src,msg="Source is not a file: %(path)s") + raise FileNotFoundError(src) if not overwrite and self.exists(dst): raise DestinationExistsError(dst) shutil.move(src_syspath, dst_syspath) @@ -597,48 +596,6 @@ class FS(object): self.remove(src) -# def _get_attr_path(self, path): -# if self.isdir(path): -# return pathjoin(path, '.xattrs.') -# else: -# dir_path, file_path = pathsplit(path) -# return pathjoin(dir_path, '.xattrs.'+file_path) -# -# def _get_attr_dict(self, path): -# attr_path = self._get_attr_path(path) -# if self.exists(attr_path): -# return pickle.loads(self.getcontents(attr_path)) -# else: -# return {} -# -# def _set_attr_dict(self, path, attrs): -# attr_path = self._get_attr_path(path) -# self.setcontents(self._get_attr_path(path), pickle.dumps(attrs)) -# -# def setxattr(self, path, key, value): -# attrs = self._get_attr_dict(path) -# attrs[key] = value -# self._set_attr_dict(path, attrs) -# -# def getxattr(self, path, key, default): -# attrs = self._get_attr_dict(path) -# return attrs.get(key, default) -# -# def delxattr(self, path, key): -# attrs = self._get_attr_dict(path) -# try: -# del attrs[key] -# except KeyError: -# pass -# self._set_attr_dict(path, attrs) -# -# def listxattrs(self, path): -# attrs = self._get_attr_dict(path) -# return self._get_attr_dict(path).keys() -# -# def getxattrs(self, path): -# return dict( [(k, self.getxattr(path, k)) for k in self.listxattrs(path)] ) - def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384): """Moves a directory from one location to another. diff --git a/fs/errors.py b/fs/errors.py index b216c64..e3eb618 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -30,7 +30,7 @@ class PathError(FSError): def __init__(self,path,**kwds): self.path = path - super(ResourceError,self).__init__(**kwds) + super(PathError,self).__init__(**kwds) class OperationFailedError(FSError): @@ -166,66 +166,31 @@ class OSFS(FS): return stats.st_size -# def setxattr(self, path, key, value): -# self._lock.acquire() -# try: -# if xattr is None: -# return FS.setxattr(self, path, key, value) -# try: -# xattr.xattr(self.getsyspath(path))[key]=value -# except IOError, e: -# if e.errno == 95: -# return FS.setxattr(self, path, key, value) -# else: -# raise OperationFailedError('XATTR_FAILED', path, details=e) -# finally: -# self._lock.release() -# -# def getxattr(self, path, key, default=None): -# self._lock.acquire() -# try: -# if xattr is None: -# return FS.getxattr(self, path, key, default) -# try: -# return xattr.xattr(self.getsyspath(path)).get(key) -# except IOError, e: -# if e.errno == 95: -# return FS.getxattr(self, path, key, default) -# else: -# raise OperationFailedError('XATTR_FAILED', path, details=e) -# finally: -# self._lock.release() -# -# def removexattr(self, path, key): -# self._lock.acquire() -# try: -# if xattr is None: -# return FS.removexattr(self, path, key) -# try: -# del xattr.xattr(self.getsyspath(path))[key] -# except KeyError: -# pass -# except IOError, e: -# if e.errono == 95: -# return FS.removexattr(self, path, key) -# else: -# raise OperationFailedError('XATTR_FAILED', path, details=e) -# finally: -# self._lock.release() -# -# def listxattrs(self, path): -# self._lock.acquire() -# try: -# if xattr is None: -# return FS.listxattrs(self, path) -# try: -# return xattr.xattr(self.getsyspath(path)).keys() -# except IOError, e: -# if errono == 95: -# return FS.listxattrs(self, path) -# else: -# raise OperationFailedError('XATTR_FAILED', path, details=e) -# finally: -# self._lock.release() -# + # Provide native xattr support if available + if xattr: + def setxattr(self, path, key, value): + try: + xattr.xattr(self.getsyspath(path))[key]=value + except IOError, e: + raise OperationFailedError('set extended attribute', path=path, details=e) + + def getxattr(self, path, key, default=None): + try: + return xattr.xattr(self.getsyspath(path)).get(key,default) + except IOError, e: + raise OperationFailedError('get extended attribute', path=path, details=e) + + def delxattr(self, path, key): + try: + del xattr.xattr(self.getsyspath(path))[key] + except KeyError: + pass + except IOError, e: + raise OperationFailedError('delete extended attribute', path=path, details=e) + + def xattrs(self, path): + try: + return xattr.xattr(self.getsyspath(path)).keys() + except IOError, e: + raise OperationFailedError('list extended attributes', path=path, details=e) diff --git a/fs/tests.py b/fs/tests.py index 9fd9a26..0f2a0d9 100644 --- a/fs/tests.py +++ b/fs/tests.py @@ -18,21 +18,19 @@ from helpers import * #################### -class BaseFSTestCases: +class FSTestCases: """Base suite of testcases for filesystem implementations. Any FS subclass should be capable of passing all of these tests. To apply the tests to your own FS implementation, simply subclass - BaseFSTestCase and have the setUp method set self.fs to an instance + FSTestCase and have the setUp method set self.fs to an instance of your FS implementation. - Note that you'll also need to subclass + Note that you'll also need to subclass unittest.TestCase - this class + is designed as a mixin so that it's not picked up by test tools such + as nose. """ - def setUp(self): - import nose - raise nose.SkipTest("this is an abstract base class") - def check(self, p): """Check that a file exists within self.fs""" return self.fs.exists(p) @@ -443,6 +441,76 @@ class BaseFSTestCases: +class XAttrTestCases: + """Testcases for filesystems providing extended attribute support.""" + + def test_getsetdel(self): + def do_getsetdel(p): + self.assertEqual(self.fs.getxattr(p,"xattr1"),None) + self.fs.setxattr(p,"xattr1","value1") + self.assertEqual(self.fs.getxattr(p,"xattr1"),"value1") + self.fs.delxattr(p,"xattr1") + self.assertEqual(self.fs.getxattr(p,"xattr1"),None) + self.fs.createfile("test.txt","hello") + do_getsetdel("test.txt") + self.assertRaises(fs.ResourceNotFoundError,self.fs.getxattr,"test2.txt","xattr1") + self.fs.makedir("mystuff") + self.fs.createfile("/mystuff/test.txt","") + do_getsetdel("mystuff") + do_getsetdel("mystuff/test.txt") + + def test_list_xattrs(self): + def do_list(p): + self.assertEquals(self.fs.xattrs(p),[]) + self.fs.setxattr(p,"xattr1","value1") + self.assertEquals(self.fs.xattrs(p),["xattr1"]) + self.fs.setxattr(p,"attr2","value2") + self.assertEquals(sorted(self.fs.xattrs(p)),["attr2","xattr1"]) + self.fs.delxattr(p,"xattr1") + self.assertEquals(self.fs.xattrs(p),["attr2"]) + self.fs.delxattr(p,"attr2") + self.assertEquals(self.fs.xattrs(p),[]) + self.fs.createfile("test.txt","hello") + do_list("test.txt") + self.fs.makedir("mystuff") + self.fs.createfile("/mystuff/test.txt","") + do_list("mystuff") + do_list("mystuff/test.txt") + + def test_copy_xattrs(self): + self.fs.createfile("a.txt","content") + self.fs.setxattr("a.txt","myattr","myvalue") + self.fs.setxattr("a.txt","testattr","testvalue") + self.fs.makedir("stuff") + self.fs.copy("a.txt","stuff/a.txt") + self.assertTrue(self.fs.exists("stuff/a.txt")) + self.assertEquals(self.fs.getxattr("stuff/a.txt","myattr"),"myvalue") + self.assertEquals(self.fs.getxattr("stuff/a.txt","testattr"),"testvalue") + self.assertEquals(self.fs.getxattr("a.txt","myattr"),"myvalue") + self.assertEquals(self.fs.getxattr("a.txt","testattr"),"testvalue") + self.fs.setxattr("stuff","dirattr","a directory") + self.fs.copydir("stuff","stuff2") + self.assertEquals(self.fs.getxattr("stuff2/a.txt","myattr"),"myvalue") + self.assertEquals(self.fs.getxattr("stuff2/a.txt","testattr"),"testvalue") + self.assertEquals(self.fs.getxattr("stuff2","dirattr"),"a directory") + self.assertEquals(self.fs.getxattr("stuff","dirattr"),"a directory") + + def test_move_xattrs(self): + self.fs.createfile("a.txt","content") + self.fs.setxattr("a.txt","myattr","myvalue") + self.fs.setxattr("a.txt","testattr","testvalue") + self.fs.makedir("stuff") + self.fs.move("a.txt","stuff/a.txt") + self.assertTrue(self.fs.exists("stuff/a.txt")) + self.assertEquals(self.fs.getxattr("stuff/a.txt","myattr"),"myvalue") + self.assertEquals(self.fs.getxattr("stuff/a.txt","testattr"),"testvalue") + self.fs.setxattr("stuff","dirattr","a directory") + self.fs.movedir("stuff","stuff2") + self.assertEquals(self.fs.getxattr("stuff2/a.txt","myattr"),"myvalue") + self.assertEquals(self.fs.getxattr("stuff2/a.txt","testattr"),"testvalue") + self.assertEquals(self.fs.getxattr("stuff2","dirattr"),"a directory") + + #################### @@ -577,7 +645,7 @@ class TestObjectTree(unittest.TestCase): import osfs import os -class TestOSFS(unittest.TestCase,BaseFSTestCases): +class TestOSFS(unittest.TestCase,FSTestCases): def setUp(self): self.temp_dir = tempfile.mkdtemp("fstest") @@ -591,7 +659,7 @@ class TestOSFS(unittest.TestCase,BaseFSTestCases): -class TestSubFS(unittest.TestCase,BaseFSTestCases): +class TestSubFS(unittest.TestCase,FSTestCases): def setUp(self): self.temp_dir = tempfile.mkdtemp("fstest") @@ -609,14 +677,14 @@ class TestSubFS(unittest.TestCase,BaseFSTestCases): import memoryfs -class TestMemoryFS(unittest.TestCase,BaseFSTestCases): +class TestMemoryFS(unittest.TestCase,FSTestCases): def setUp(self): self.fs = memoryfs.MemoryFS() import mountfs -class TestMountFS(unittest.TestCase,BaseFSTestCases): +class TestMountFS(unittest.TestCase,FSTestCases): def setUp(self): self.mount_fs = mountfs.MountFS() @@ -632,7 +700,7 @@ class TestMountFS(unittest.TestCase,BaseFSTestCases): import tempfs -class TestTempFS(unittest.TestCase,BaseFSTestCases): +class TestTempFS(unittest.TestCase,FSTestCases): def setUp(self): self.fs = tempfs.TempFS() @@ -648,7 +716,7 @@ class TestTempFS(unittest.TestCase,BaseFSTestCases): import s3fs -class TestS3FS(unittest.TestCase,BaseFSTestCases): +class TestS3FS(unittest.TestCase,FSTestCases): bucket = "test-s3fs.rfk.id.au" @@ -673,7 +741,7 @@ class TestS3FS(unittest.TestCase,BaseFSTestCases): import rpcfs import socket import threading -class TestRPCFS(unittest.TestCase,BaseFSTestCases): +class TestRPCFS(unittest.TestCase,FSTestCases): def setUp(self): self.port = 8000 @@ -708,6 +776,22 @@ class TestRPCFS(unittest.TestCase,BaseFSTestCases): pass +from fs.wrappers.xattr import SimulateXAttr +class TestSimulateXAttr(unittest.TestCase,FSTestCases,XAttrTestCases): + + def setUp(self): + self.fs = SimulateXAttr(tempfs.TempFS()) + + def tearDown(self): + td = self.fs._temp_dir + self.fs.close() + self.assert_(not os.path.exists(td)) + + def check(self, p): + td = self.fs._temp_dir + return os.path.exists(os.path.join(td, makerelative(p))) + + #################### diff --git a/fs/wrappers/__init__.py b/fs/wrappers/__init__.py new file mode 100644 index 0000000..4221d78 --- /dev/null +++ b/fs/wrappers/__init__.py @@ -0,0 +1,130 @@ +""" + + fs.wrappers: clases to transform files in a filesystem + +This sub-module provides facilities for easily wrapping an FS object inside +some sort of transformation - for example, to transparently compress or encrypt +files. + +""" + +from fs.base import FS + +class FSWrapper(FS): + """FS that wraps another FS, providing translation etc. + + This class allows simple transforms to be applied to the names + and/or contents of files in an FS. It's particularly handy in + conjunction with wrappers from the "filelike" module. + """ + + def __init__(self,fs): + super(FSWrapper,self).__init__() + self.wrapped_fs = fs + + def _file_wrap(self,f,mode): + """Apply wrapping to an opened file.""" + return f + + def _encode_name(self,name): + """Encode path component for the underlying FS.""" + return name + + def _decode_name(self,name): + """Decode path component from the underlying FS.""" + return name + + def _encode(self,path): + """Encode path for the underlying FS.""" + names = path.split("/") + e_names = [] + for name in names: + if name == "": + e_names.append("") + else: + e_names.append(self._encode_name(name)) + return "/".join(e_names) + + def _decode(self,path): + """Decode path from the underlying FS.""" + names = path.split("/") + d_names = [] + for name in names: + if name == "": + d_names.append("") + else: + d_names.append(self._decode_name(name)) + return "/".join(d_names) + + def _adjust_mode(self,mode): + """Adjust the mode used to open a file in the underlying FS. + + This method takes the mode given when opening a file, and should + return a two-tuple giving the mode to be used in this FS as first + item, and the mode to be used in the underlying FS as the second. + """ + return (mode,mode) + + def getsyspath(self,path,allow_none=False): + return self.wrapped_fs.getsyspath(self._encode(path),allow_none) + + def hassyspath(self,path): + return self.wrapped_fs.hassyspath(self._encode(path)) + + def open(self,path,mode="r"): + (mode,wmode) = self._adjust_mode(mode) + f = self.wrapped_fs.open(self._encode(path),wmode) + return self._file_wrap(f,mode) + + def exists(self,path): + return self.wrapped_fs.exists(self._encode(path)) + + def isdir(self,path): + return self.wrapped_fs.isdir(self._encode(path)) + + def isfile(self,path): + return self.wrapped_fs.isfile(self._encode(path)) + + def listdir(self,path="",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False): + entries = [] + for name in self.wrapped_fs.listdir(self._encode(path),wildcard=None,full=full,absolute=absolute,dirs_only=dirs_only,files_only=files_only): + entries.append(self._decode(name)) + return self._listdir_helper(path,entries,wildcard=wildcard,full=False,absolute=False,dirs_only=False,files_only=False) + + def makedir(self,path,*args,**kwds): + return self.wrapped_fs.makedir(self._encode(path),*args,**kwds) + + def remove(self,path): + return self.wrapped_fs.remove(self._encode(path)) + + def removedir(self,path,*args,**kwds): + return self.wrapped_fs.removedir(self._encode(path),*args,**kwds) + + def rename(self,src,dst): + return self.wrapped_fs.rename(self._encode(src),self._encode(dst)) + + def getinfo(self,path): + return self.wrapped_fs.getinfo(self._encode(path)) + + def desc(self,path): + return self.wrapped_fs.desc(self._encode(path)) + + def copy(self,src,dst,overwrite=False,chunk_size=16384): + return self.wrapped_fs.copy(self._encode(src),self._encode(dst),overwrite,chunk_size) + + def move(self,src,dst,overwrite=False,chunk_size=16384): + return self.wrapped_fs.move(self._encode(src),self._encode(dst),overwrite,chunk_size) + + def movedir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384): + return self.wrapped_fs.movedir(self._encode(src),self._encode(dst),overwrite,ignore_errors,chunk_size) + + def copydir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384): + return self.wrapped_fs.copydir(self._encode(src),self._encode(dst),overwrite,ignore_errors,chunk_size) + + def __getattr__(self,attr): + return getattr(self.wrapped_fs,attr) + + def close(self): + if hasattr(self.wrapped_fs,"close"): + self.wrapped_fs.close() + diff --git a/fs/wrappers/xattr.py b/fs/wrappers/xattr.py new file mode 100644 index 0000000..3eb5820 --- /dev/null +++ b/fs/wrappers/xattr.py @@ -0,0 +1,115 @@ +""" + + fs.wrappers.xattr: FS wrapper for simulating extended-attribute support. + +""" + +try: + import cPickle as pickle +except ImportError: + import pickle + +from fs.helpers import * +from fs.errors import * +from fs.wrappers import FSWrapper + +class SimulateXAttr(FSWrapper): + """FS wrapper class that simulates xattr support. + + For each file in the underlying FS, this class managed a corresponding + '.xattr' file containing its extended attributes. + """ + + def _get_attr_path(self, path): + """Get the path of the file containing xattrs for the given path.""" + if self.wrapped_fs.isdir(path): + return pathjoin(path, '.xattrs.') + else: + dir_path, file_name = pathsplit(path) + return pathjoin(dir_path, '.xattrs.'+file_name) + + def _is_attr_path(self, path): + """Check whether the given path references an xattrs file.""" + _,name = pathsplit(path) + if name.startswith(".xattrs."): + return True + return False + + def _get_attr_dict(self, path): + """Retrieve the xattr dictionary for the given path.""" + attr_path = self._get_attr_path(path) + if self.wrapped_fs.exists(attr_path): + return pickle.loads(self.wrapped_fs.getcontents(attr_path)) + else: + return {} + + def _set_attr_dict(self, path, attrs): + """Store the xattr dictionary for the given path.""" + attr_path = self._get_attr_path(path) + self.wrapped_fs.setcontents(self._get_attr_path(path), pickle.dumps(attrs)) + + def setxattr(self, path, key, value): + """Set an extended attribute on the given path.""" + if not self.exists(path): + raise ResourceNotFoundError(path) + attrs = self._get_attr_dict(path) + attrs[key] = value + self._set_attr_dict(path, attrs) + + def getxattr(self, path, key, default=None): + """Retrieve an extended attribute for the given path.""" + if not self.exists(path): + raise ResourceNotFoundError(path) + attrs = self._get_attr_dict(path) + return attrs.get(key, default) + + def delxattr(self, path, key): + if not self.exists(path): + raise ResourceNotFoundError(path) + attrs = self._get_attr_dict(path) + try: + del attrs[key] + except KeyError: + pass + self._set_attr_dict(path, attrs) + + def xattrs(self,path): + """List all the extended attribute keys set on the given path.""" + if not self.exists(path): + raise ResourceNotFoundError(path) + return self._get_attr_dict(path).keys() + + def _encode(self,path): + """Prevent requests for operations on .xattr files.""" + if self._is_attr_path(path): + raise PathError(path,msg="Paths cannot contain '.xattrs.': %(path)s") + return path + + def _decode(self,path): + return path + + def listdir(self,path="",**kwds): + """Prevent .xattr from appearing in listings.""" + entries = self.wrapped_fs.listdir(path,**kwds) + return [e for e in entries if not self._is_attr_path(e)] + + def copy(self,src,dst,**kwds): + """Ensure xattrs are copied when copying a file.""" + self.wrapped_fs.copy(self._encode(src),self._encode(dst),**kwds) + s_attr_file = self._get_attr_path(src) + d_attr_file = self._get_attr_path(dst) + try: + self.wrapped_fs.copy(s_attr_file,d_attr_file,overwrite=True) + except ResourceNotFoundError,e: + pass + + def move(self,src,dst,**kwds): + """Ensure xattrs are preserved when moving a file.""" + self.wrapped_fs.move(self._encode(src),self._encode(dst),**kwds) + s_attr_file = self._get_attr_path(src) + d_attr_file = self._get_attr_path(dst) + try: + self.wrapped_fs.move(s_attr_file,d_attr_file,overwrite=True) + except ResourceNotFoundError: + pass + |