updated test suite now passing

git-svn-id: http://pyfilesystem.googlecode.com/svn/branches/rfk-ideas@138 67cdc799-7952-0410-af00-57a81ceafa0f

8 files changed, 123 insertions, 92 deletions

diff --git a/fs/__init__.py b/fs/__init__.py
index f843da2..fcd45e3 100644
--- a/fs/__init__.py
+++ b/fs/__init__.py
@@ -16,4 +16,5 @@ __all__ = ['memoryfs',
            'utils',
            'zipfs',
            'helpers',
-           'tempfs']
\ No newline at end of file
+           'tempfs']
+
diff --git a/fs/errors.py b/fs/errors.py
index 35b038c..b216c64 100644
--- a/fs/errors.py
+++ b/fs/errors.py
@@ -17,7 +17,7 @@ class FSError(Exception):
 
     def __str__(self):
         keys = dict((k,str(v)) for k,v in self.__dict__.iteritems())
-        return "FSError: " + (self.msg % keys)
+        return self.msg % keys
 
     def __unicode__(self):
         return unicode(str(self))
diff --git a/fs/helpers.py b/fs/helpers.py
index 7d86ba2..d79e4e1 100644
--- a/fs/helpers.py
+++ b/fs/helpers.py
@@ -110,7 +110,7 @@ def pathjoin(*paths):
              relpaths.append(p)
 
     path = normpath("/".join(relpaths))
-    if absolute:
+    if absolute and not path.startswith("/"):
         path = "/" + path
     return path
 
diff --git a/fs/osfs.py b/fs/osfs.py
index 613b50e..7b675cb 100644
--- a/fs/osfs.py
+++ b/fs/osfs.py
@@ -124,7 +124,7 @@ class OSFS(FS):
         if recursive:
             try:
                 self.removedir(dirname(path),recursive=True)
-            except OperationFailedError:
+            except DirectoryNotEmptyError:
                 pass
 
     def rename(self, src, dst):
diff --git a/fs/rpcfs.py b/fs/rpcfs.py
index 6141aae..880379b 100644
--- a/fs/rpcfs.py
+++ b/fs/rpcfs.py
@@ -25,19 +25,16 @@ from fs.base import *
 
 from StringIO import StringIO
 
-
-class ObjProxy:
-    """Simple object proxy allowing us to replace read-only attributes.
-
-    This is used to put a modified 'close' method on files returned by
-    open(), such that they will be uploaded to the server when closed.
-    """
-
-    def __init__(self,obj):
-        self._obj = obj
-
-    def __getattr__(self,attr):
-        return getattr(self._obj,attr)
+if hasattr(StringIO,"__exit__"):
+    class StringIO(StringIO):
+        pass
+else:
+    class StringIO(StringIO):
+        def __enter__(self):
+            return self
+        def __exit__(self,exc_type,exc_value,traceback):
+            self.close()
+            return False
 
 
 def re_raise_faults(func):
@@ -119,18 +116,34 @@ class RPCFS(FS):
         object along with the 'transport' argument if it is provided.
         """
         self.uri = uri
-        if transport is not None:
-            proxy = xmlrpclib.ServerProxy(uri,transport,allow_none=True)
+        self._transport = transport
+        self.proxy = self._make_proxy()
+
+    def _make_proxy(self):
+        kwds = dict(allow_none=True)
+        if self._transport is not None:
+            proxy = xmlrpclib.ServerProxy(self.uri,self._transport,**kwds)
         else:
-            proxy = xmlrpclib.ServerProxy(uri,allow_none=True)
-        self.proxy = ReRaiseFaults(proxy)
+            proxy = xmlrpclib.ServerProxy(self.uri,**kwds)
+        return ReRaiseFaults(proxy)
 
     def __str__(self):
         return '<RPCFS: %s>' % (self.uri,)
 
-    __repr__ = __str__
+    def __getstate__(self):
+        state = super(RPCFS,self).__getstate__()
+        try:
+            del state['proxy']
+        except KeyError:
+            pass
+        return state
+
+    def __setstate__(self,state):
+        for (k,v) in state.iteritems():
+            self.__dict__[k] = v
+        self.proxy = self._make_proxy()
 
-    def open(self,path,mode):
+    def open(self,path,mode="r"):
         # TODO: chunked transport of large files
         if "w" in mode:
             self.proxy.set_contents(path,xmlrpclib.Binary(""))
@@ -139,13 +152,13 @@ class RPCFS(FS):
                 data = self.proxy.get_contents(path).data
             except IOError:
                 if "w" not in mode and "a" not in mode:
-                    raise ResourceNotFoundError("NO_FILE",path)
+                    raise FileNotFoundError(path)
                 if not self.isdir(dirname(path)):
-                    raise OperationFailedError("OPEN_FAILED", path,msg="Parent directory does not exist")
+                    raise ParentDirectoryMissingError(path)
                 self.proxy.set_contents(path,xmlrpclib.Binary(""))
         else:
             data = ""
-        f = ObjProxy(StringIO(data))
+        f = StringIO(data)
         if "a" not in mode:
             f.seek(0,0)
         else:
@@ -154,6 +167,7 @@ class RPCFS(FS):
         oldclose = f.close
         def newflush():
             oldflush()
+            print "SENDING:", f.getvalue()
             self.proxy.set_contents(path,xmlrpclib.Binary(f.getvalue()))
         def newclose():
             f.flush()
@@ -171,11 +185,11 @@ class RPCFS(FS):
     def isfile(self,path):
         return self.proxy.isfile(path)
 
-    def listdir(self,path="./",wildcard=None,full=False,absolute=False,hidden=True,dirs_only=False,files_only=False):
-        return self.proxy.listdir(path,wildcard,full,absolute,hidden,dirs_only,files_only)
+    def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
+        return self.proxy.listdir(path,wildcard,full,absolute,dirs_only,files_only)
 
-    def makedir(self,path,mode=0777,recursive=False,allow_recreate=False):
-        return self.proxy.makedir(path,mode,recursive,allow_recreate)
+    def makedir(self,path,recursive=False,allow_recreate=False):
+        return self.proxy.makedir(path,recursive,allow_recreate)
 
     def remove(self,path):
         return self.proxy.remove(path)
@@ -214,7 +228,7 @@ class RPCFS(FS):
 class RPCFSInterface(object):
     """Wrapper to expose an FS via a XML-RPC compatible interface.
 
-    The only real trick is using xmlrpclib.Binary objects to trasnport
+    The only real trick is using xmlrpclib.Binary objects to transport
     the contents of files.
     """
 
@@ -237,11 +251,11 @@ class RPCFSInterface(object):
     def isfile(self,path):
         return self.fs.isfile(path)
 
-    def listdir(self,path="./",wildcard=None,full=False,absolute=False,hidden=True,dirs_only=False,files_only=False):
-        return list(self.fs.listdir(path,wildcard,full,absolute,hidden,dirs_only,files_only))
+    def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
+        return list(self.fs.listdir(path,wildcard,full,absolute,dirs_only,files_only))
 
-    def makedir(self,path,mode=0777,recursive=False,allow_recreate=False):
-        return self.fs.makedir(path,mode,recursive,allow_recreate)
+    def makedir(self,path,recursive=False,allow_recreate=False):
+        return self.fs.makedir(path,recursive,allow_recreate)
 
     def remove(self,path):
         return self.fs.remove(path)
diff --git a/fs/s3fs.py b/fs/s3fs.py
index 8517278..145a604 100644
--- a/fs/s3fs.py
+++ b/fs/s3fs.py
@@ -38,10 +38,10 @@ class S3FS(FS):
         PATH_MAX = None
         NAME_MAX = None
 
-    def __init__(self, bucket, prefix="", aws_access_key=None, aws_secret_key=None, separator="/", thread_syncronize=True,key_sync_timeout=1):
+    def __init__(self, bucket, prefix="", aws_access_key=None, aws_secret_key=None, separator="/", thread_synchronize=True,key_sync_timeout=1):
         """Constructor for S3FS objects.
 
-        S3FS objects required the name of the S3 bucket in which to store
+        S3FS objects require the name of the S3 bucket in which to store
         files, and can optionally be given a prefix under which the files
         shoud be stored.  The AWS public and private keys may be specified
         as additional arguments; if they are not specified they will be
@@ -63,12 +63,13 @@ class S3FS(FS):
         self._separator = separator
         self._key_sync_timeout = key_sync_timeout
         # Normalise prefix to this form: path/to/files/
+        prefix = normpath(prefix)
         while prefix.startswith(separator):
             prefix = prefix[1:]
         if not prefix.endswith(separator) and prefix != "":
             prefix = prefix + separator
         self._prefix = prefix
-        FS.__init__(self, thread_syncronize=thread_syncronize)
+        FS.__init__(self, thread_synchronize=thread_synchronize)
 
     #  Make _s3conn and _s3bukt properties that are created on demand,
     #  since they cannot be stored during pickling.
@@ -115,15 +116,12 @@ class S3FS(FS):
 
     def _s3path(self,path):
         """Get the absolute path to a file stored in S3."""
-        path = self._prefix + path
-        path = self._separator.join(self._pathbits(path))
-        return path
-
-    def _pathbits(self,path):
-        """Iterator over path components."""
-        for bit in path.split("/"):
-            if bit and bit != ".":
-              yield bit
+        path = makerelative(path)
+        path = self._separator.join(iteratepath(path))
+        s3path = self._prefix + path
+        if s3path[-1] == self._separator:
+            s3path = s3path[:-1]
+        return s3path
 
     def _sync_key(self,k):
         """Synchronise on contents of the given key.
@@ -177,9 +175,9 @@ class S3FS(FS):
         if k is None:
             # Create the file if it's missing
             if "w" not in mode and "a" not in mode:
-                raise ResourceNotFoundError("NO_FILE",path)
+                raise FileNotFoundError(path)
             if not self.isdir(dirname(path)):
-                raise OperationFailedError("OPEN_FAILED", path,msg="Parent directory does not exist")
+                raise ParentDirectoryMissingError(path)
             k = self._sync_set_contents(s3path,"")
         else:
             # Get the file contents into the tempfile.
@@ -258,7 +256,7 @@ class S3FS(FS):
           return True
         return False
 
-    def listdir(self,path="./",wildcard=None,full=False,absolute=False,hidden=True,dirs_only=False,files_only=False):
+    def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
         """List contents of a directory."""
         s3path = self._s3path(path) + self._separator
         if s3path == "/":
@@ -278,10 +276,12 @@ class S3FS(FS):
                 paths.append(nm)
         if not isDir:
             if s3path != self._prefix:
-                raise OperationFailedError("LISTDIR_FAILED",path)
-        return self._listdir_helper(path,paths,wildcard,full,absolute,hidden,dirs_only,files_only)
+                if self.isfile(path):
+                    raise ResourceInvalidError(path,msg="that's not a directory: %(path)s")
+                raise DirectoryNotFoundError(path)
+        return self._listdir_helper(path,paths,wildcard,full,absolute,dirs_only,files_only)
 
-    def _listdir_helper(self,path,paths,wildcard,full,absolute,hidden,dirs_only,files_only):
+    def _listdir_helper(self,path,paths,wildcard,full,absolute,dirs_only,files_only):
         """Modify listdir helper to avoid additional calls to the server."""
         if dirs_only and files_only:
             raise ValueError("dirs_only and files_only can not both be True")
@@ -299,17 +299,14 @@ class S3FS(FS):
             match = fnmatch.fnmatch
             paths = [p for p in paths if match(p, wildcard)]
 
-        if not hidden:
-            paths = [p for p in paths if not self.ishidden(p)]
-
         if full:
             paths = [pathjoin(path, p) for p in paths]
         elif absolute:
-            paths = [self._abspath(pathjoin(path, p)) for p in paths]
+            paths = [abspath(pathjoin(path, p)) for p in paths]
 
         return paths
         
-    def makedir(self,path,mode=0777,recursive=False,allow_recreate=False):
+    def makedir(self,path,recursive=False,allow_recreate=False):
         """Create a directory at the given path.
 
         The 'mode' argument is accepted for compatability with the standard
@@ -320,8 +317,8 @@ class S3FS(FS):
         if s3pathD == self._prefix:
             if allow_recreate:
                 return
-            raise OperationFailedError("MAKEDIR_FAILED", path, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")
-        s3pathP = self._s3path(dirname(path[:-1])) + self._separator
+            raise DestinationExistsError(path, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")
+        s3pathP = self._s3path(dirname(path)) + self._separator
         # Check various preconditions using list of parent dir
         ks = self._s3bukt.list(prefix=s3pathP,delimiter=self._separator)
         if s3pathP == self._prefix:
@@ -333,26 +330,33 @@ class S3FS(FS):
                 parentExists = True
             if k.name == s3path:
                 # It's already a file
-                raise OperationFailedError("MAKEDIR_FAILED", path, msg="Can not create a directory that already exists: %(path)s")
+                raise ResourceInvalidError(path, msg="Destination exists as a regular file: %(path)s")
             if k.name == s3pathD:
                 # It's already a directory
                 if allow_recreate:
                     return
-                raise OperationFailedError("MAKEDIR_FAILED", path, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")
+                raise DestinationExistsError(path, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")
         # Create parent if required
         if not parentExists:
             if recursive:
-                self.makedir(dirname(path[:-1]),mode,recursive,allow_recreate)
+                self.makedir(dirname(path),recursive,allow_recreate)
             else:
-                raise OperationFailedError("MAKEDIR_FAILED",path, msg="Parent directory does not exist: %(path)s")
+                raise ParentDirectoryMissingError(path, msg="Parent directory does not exist: %(path)s")
         # Create an empty file representing the directory
         # TODO: is there some standard scheme for representing empty dirs?
         self._sync_set_contents(s3pathD,"")
 
     def remove(self,path):
         """Remove the file at the given path."""
-        # TODO: This will fail silently if the key doesn't exist
         s3path = self._s3path(path)
+        ks = self._s3bukt.list(prefix=s3path,delimiter=self._separator)
+        for k in ks:
+            if k.name == s3path:
+                break
+            if k.name.startswith(s3path + "/"):
+                raise ResourceInvalidError(path,msg="that's not a file: %(path)s")
+        else:
+            raise FileNotFoundError(path)
         self._s3bukt.delete_key(s3path)
         k = self._s3bukt.get_key(s3path)
         while k:
@@ -368,17 +372,23 @@ class S3FS(FS):
         else:
             ks = self._s3bukt.list(prefix=s3path,delimiter=self._separator)
         # Fail if the directory is not empty, or remove them if forced
+        found = False
         for k in ks:
+            found = True
             if k.name != s3path:
                 if not force:
-                    raise OperationFailedError("REMOVEDIR_FAILED",path)
+                    raise DirectoryNotEmptyError(path)
                 self._s3bukt.delete_key(k.name)
+        if not found:
+            if self.isfile(path):
+                raise ResourceInvalidError(path,msg="removedir() called on a regular file: %(path)s")
+            raise DirectoryNotFoundError(path)
         self._s3bukt.delete_key(s3path)
-        if recursive:
+        if recursive and path not in ("","/"):
             pdir = dirname(path)
             try:
                 self.removedir(pdir,recursive=True,force=False)
-            except OperationFailedError:
+            except DirectoryNotEmptyError:
                 pass
         
     def rename(self,src,dst):
@@ -419,7 +429,7 @@ class S3FS(FS):
             # It exists as a regular file
             if k.name == s3path_dst:
                 if not overwrite:
-                    raise DestinationExistsError("COPYFILE_FAILED",src,dst,msg="Destination file exists: %(path2)s")
+                    raise DestinationExistsError(dst)
                 dstOK = True
                 break
             # Check if it refers to a directory.  If so, we copy *into* it.
@@ -431,14 +441,14 @@ class S3FS(FS):
                 s3path_dst = s3path_dstD + nm
                 dstOK = True
         if not dstOK and not self.isdir(dirname(dst)):
-            raise OperationFailedError("COPYFILE_FAILED",src,dst,msg="Destination directory does not exist")
+            raise ParentDirectoryMissingError(dst,msg="Destination directory does not exist: %(path)s")
         # OK, now we can copy the file.
         s3path_src = self._s3path(src)
         try:
             self._s3bukt.copy_key(s3path_dst,self._bucket_name,s3path_src)
         except S3ResponseError, e:
             if "404 Not Found" in str(e):
-                raise ResourceInvalid("WRONG_TYPE", src, msg="Source is not a file: %(path)s")
+                raise ResourceInvalidError(src, msg="Source is not a file: %(path)s")
             raise e
         else:
             k = self._s3bukt.get_key(s3path_dst)
diff --git a/fs/tests.py b/fs/tests.py
index 64fcd8b..9fd9a26 100644
--- a/fs/tests.py
+++ b/fs/tests.py
@@ -18,14 +18,20 @@ from helpers import *
 ####################
 
 
-class BaseFSTestCase(unittest.TestCase):
+class BaseFSTestCases:
     """Base suite of testcases for filesystem implementations.
 
     Any FS subclass should be capable of passing all of these tests.
     To apply the tests to your own FS implementation, simply subclass
     BaseFSTestCase and have the setUp method set self.fs to an instance
     of your FS implementation.
+
+    Note that you'll also need to subclass
     """
+
+    def setUp(self):
+        import nose
+        raise nose.SkipTest("this is an abstract base class")
         
     def check(self, p):
         """Check that a file exists within self.fs"""
@@ -571,7 +577,7 @@ class TestObjectTree(unittest.TestCase):
 import osfs
 import os
 
-class TestOSFS(BaseFSTestCase):
+class TestOSFS(unittest.TestCase,BaseFSTestCases):
 
     def setUp(self):
         self.temp_dir = tempfile.mkdtemp("fstest")
@@ -585,7 +591,7 @@ class TestOSFS(BaseFSTestCase):
 
 
 
-class TestSubFS(BaseFSTestCase):
+class TestSubFS(unittest.TestCase,BaseFSTestCases):
 
     def setUp(self):
         self.temp_dir = tempfile.mkdtemp("fstest")
@@ -603,14 +609,14 @@ class TestSubFS(BaseFSTestCase):
 
 
 import memoryfs
-class TestMemoryFS(BaseFSTestCase):
+class TestMemoryFS(unittest.TestCase,BaseFSTestCases):
 
     def setUp(self):
         self.fs = memoryfs.MemoryFS()
 
 
 import mountfs
-class TestMountFS(BaseFSTestCase):
+class TestMountFS(unittest.TestCase,BaseFSTestCases):
 
     def setUp(self):
         self.mount_fs = mountfs.MountFS()
@@ -626,7 +632,7 @@ class TestMountFS(BaseFSTestCase):
 
 
 import tempfs
-class TestTempFS(BaseFSTestCase):
+class TestTempFS(unittest.TestCase,BaseFSTestCases):
 
     def setUp(self):
         self.fs = tempfs.TempFS()
@@ -642,7 +648,7 @@ class TestTempFS(BaseFSTestCase):
 
 
 import s3fs
-class TestS3FS(BaseFSTestCase):
+class TestS3FS(unittest.TestCase,BaseFSTestCases):
 
     bucket = "test-s3fs.rfk.id.au"
 
@@ -667,7 +673,7 @@ class TestS3FS(BaseFSTestCase):
 import rpcfs
 import socket
 import threading
-class TestRPCFS(TestOSFS):
+class TestRPCFS(unittest.TestCase,BaseFSTestCases):
 
     def setUp(self):
         self.port = 8000
diff --git a/fs/zipfs.py b/fs/zipfs.py
index 90f84d4..78d60ec 100644
--- a/fs/zipfs.py
+++ b/fs/zipfs.py
@@ -50,17 +50,17 @@ class ZipFS(FS):
 
     """A FileSystem that represents a zip file."""
 
-    def __init__(self, zip_file, mode="r", compression="deflated", allowZip64=False, thread_syncronize=True):
+    def __init__(self, zip_file, mode="r", compression="deflated", allowZip64=False, thread_synchronize=True):
         """Create a FS that maps on to a zip file.
 
         zip_file -- A (system) path, or a file-like object
         mode -- Mode to open zip file: 'r' for reading, 'w' for writing or 'a' for appending
         compression -- Can be 'deflated' (default) to compress data or 'stored' to just store date
         allowZip64 -- Set to True to use zip files greater than 2 MB, default is False
-        thread_syncronize -- Set to True (default) to enable thread-safety
+        thread_synchronize -- Set to True (default) to enable thread-safety
 
         """
-        FS.__init__(self, thread_syncronize=thread_syncronize)
+        FS.__init__(self, thread_synchronize=thread_synchronize)
         if compression == "deflated":
             compression_type = ZIP_DEFLATED
         elif compression == "stored":
@@ -75,7 +75,7 @@ class ZipFS(FS):
         try:
             self.zf = ZipFile(zip_file, mode, compression_type, allowZip64)
         except IOError:
-            raise ResourceNotFoundError("NO_FILE", str(zip_file), msg="Zip file does not exist: %(path)s")
+            raise FileNotFoundError(str(zip_file), msg="Zip file does not exist: %(path)s")
         self.zip_path = str(zip_file)
 
         self.temp_fs = None
@@ -129,11 +129,11 @@ class ZipFS(FS):
 
             if 'r' in mode:
                 if self.zip_mode not in 'ra':
-                    raise OperationFailedError("OPEN_FAILED", path, msg="Zip file must be opened for reading ('r') or appending ('a')")
+                    raise OperationFailedError("open file", path=path, msg="Zip file must be opened for reading ('r') or appending ('a')")
                 try:
                     contents = self.zf.read(path)
                 except KeyError:
-                    raise ResourceNotFoundError("NO_FILE", path)
+                    raise FileNotFoundError(path)
                 return StringIO(contents)
 
             if 'w' in mode:
@@ -154,14 +154,14 @@ class ZipFS(FS):
         self._lock.acquire()
         try:
             if not self.exists(path):
-                raise ResourceNotFoundError("NO_FILE", path)
+                raise FileNotFoundError(path)
             path = normpath(path)
             try:
                 contents = self.zf.read(path)
             except KeyError:
-                raise ResourceNotFoundError("NO_FILE", path)
+                raise FileNotFoundError(path)
             except RuntimeError:
-                raise OperationFailedError("READ_FAILED", path, "Zip file must be oppened with 'r' or 'a' to read")
+                raise OperationFailedError("read file", path=path, msg="Zip file must be oppened with 'r' or 'a' to read")
             return contents
         finally:
             self._lock.release()
@@ -194,23 +194,23 @@ class ZipFS(FS):
         try:
             dirname = normpath(dirname)
             if self.zip_mode not in "wa":
-                raise OperationFailedError("MAKEDIR_FAILED", dirname, "Zip file must be opened for writing ('w') or appending ('a')")
+                raise OperationFailedError("create directory", path=dirname, msg="Zip file must be opened for writing ('w') or appending ('a')")
             if not dirname.endswith('/'):
                 dirname += '/'
             self._add_resource(dirname)
         finally:
             self._lock.release()
 
-    def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=True, dirs_only=False, files_only=False):
+    def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
 
-        return self._path_fs.listdir(path, wildcard, full, absolute, hidden, dirs_only, files_only)
+        return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only)
 
 
     def getinfo(self, path):
         self._lock.acquire()
         try:
             if not self.exists(path):
-                return ResourceNotFoundError("NO_RESOURCE", path)
+                return ResourceNotFoundError(path)
             path = normpath(path).lstrip('/')
             try:
                 zi = self.zf.getinfo(path)