diff options
-rw-r--r-- | fs/__init__.py | 6 | ||||
-rw-r--r-- | fs/errors.py | 98 | ||||
-rw-r--r-- | fs/expose/fuse/__init__.py | 11 | ||||
-rw-r--r-- | fs/expose/sftp.py | 8 | ||||
-rw-r--r-- | fs/memoryfs.py | 444 | ||||
-rw-r--r-- | fs/mountfs.py | 14 | ||||
-rw-r--r-- | fs/multifs.py | 6 | ||||
-rw-r--r-- | fs/osfs.py | 95 | ||||
-rw-r--r-- | fs/path.py | 6 | ||||
-rw-r--r-- | fs/rpcfs.py | 2 | ||||
-rw-r--r-- | fs/s3fs.py | 10 | ||||
-rw-r--r-- | fs/sftpfs.py | 69 | ||||
-rw-r--r-- | fs/tests/__init__.py | 2 | ||||
-rw-r--r-- | fs/tests/test_s3fs.py | 2 | ||||
-rw-r--r-- | fs/zipfs.py | 8 |
15 files changed, 369 insertions, 412 deletions
diff --git a/fs/__init__.py b/fs/__init__.py index 3872617..cbcd176 100644 --- a/fs/__init__.py +++ b/fs/__init__.py @@ -17,10 +17,12 @@ implementations of this interface such as: __version__ = "0.1.1dev" __author__ = "Will McGugan (will@willmcgugan.com)" -# 'base' imports * from 'path' and 'errors', so they'll -# be available here as well +# 'base' imports * from 'path' and 'errors', so their contents +# will be available here as well. from base import * +# provide these by default so people cna be 'fs.path.basename' etc. import errors import path + diff --git a/fs/errors.py b/fs/errors.py index 13ceaf2..372096a 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -4,6 +4,20 @@ """ +import sys +import errno + +try: + from functools import wraps +except ImportError: + def wraps(func): + def decorator(wfunc): + wfunc.__name__ == func.__name__ + wfunc.__doc__ == func.__doc__ + wfunc.__module__ == func.__module__ + return decorator + + class FSError(Exception): """Base exception class for the FS module.""" default_message = "Unspecified error" @@ -54,7 +68,11 @@ class RemoteConnectionError(OperationFailedError): class StorageSpaceError(OperationFailedError): """Exception raised when operations encounter storage space trouble.""" - default_message = "Unable to %(opname)s: remote connection errror" + default_message = "Unable to %(opname)s: insufficient storage space" + + +class PermissionDeniedError(OperationFailedError): + default_message = "Unable to %(opname)s: permission denied" @@ -64,6 +82,7 @@ class ResourceError(FSError): def __init__(self,path,**kwds): self.path = path + self.opname = kwds.pop("opname",None) super(ResourceError,self).__init__(**kwds) @@ -77,31 +96,11 @@ class ResourceNotFoundError(ResourceError): default_message = "Resource not found: %(path)s" -class DirectoryNotFoundError(ResourceNotFoundError): - """Exception raised when a required directory is not found.""" - default_message = "Directory not found: %(path)s" - - -class FileNotFoundError(ResourceNotFoundError): - """Exception raised when a required file is not found.""" - default_message = "File not found: %(path)s" - - class ResourceInvalidError(ResourceError): """Exception raised when a resource is the wrong type.""" default_message = "Resource is invalid: %(path)s" -class NotAFileError(ResourceError): - """Exception raised when a required file is not found.""" - default_message = "That's not a file: %(path)s" - - -class NotADirectoryError(ResourceError): - """Exception raised when a required file is not found.""" - default_message = "That's not a directory: %(path)s" - - class DestinationExistsError(ResourceError): """Exception raised when a target destination already exists.""" default_message = "Destination exists: %(path)s" @@ -121,3 +120,60 @@ class ResourceLockedError(ResourceError): """Exception raised when a resource can't be used because it is locked.""" default_message = "Resource is locked: %(path)s" + + +def convert_fs_errors(func): + """Function wrapper to convert FSError instances into OSErrors.""" + @wraps(func) + def wrapper(*args,**kwds): + try: + return func(*args,**kwds) + except ResourceNotFoundError, e: + raise OSError(errno.ENOENT,str(e)) + except ResourceInvalidError, e: + raise OSError(errno.EINVAL,str(e)) + except PermissionDeniedError, e: + raise OSError(errno.EACCESS,str(e)) + except DirectoryNotEmptyError, e: + raise OSError(errno.ENOTEMPTY,str(e)) + except DestinationExistsError, e: + raise OSError(errno.EEXIST,str(e)) + except StorageSpaceError, e: + raise OSError(errno.ENOSPC,str(e)) + except RemoteConnectionError, e: + raise OSError(errno.ENONET,str(e)) + except UnsupportedError, e: + raise OSError(errno.ENOSYS,str(e)) + except FSError, e: + raise OSError(errno.EFAULT,str(e)) + return wrapper + + +def convert_os_errors(func): + """Function wrapper to convert OSError/IOError instances into FSErrors.""" + opname = func.__name__ + @wraps(func) + def wrapper(*args,**kwds): + try: + return func(*args,**kwds) + except (OSError,IOError), e: + if not hasattr(e,"errno") or not e.errno: + raise OperationFailedError(opname,details=e) + if e.errno == errno.ENOENT: + raise ResourceNotFoundError(e.filename,opname=opname,details=e) + if e.errno == errno.ENOTEMPTY: + raise DirectoryNotEmptyError(e.filename,opname=opname,details=e) + if e.errno == errno.EEXIST: + raise DestinationExistsError(e.filename,opname=opname,details=e) + if e.errno == 183: # some sort of win32 equivalent to EEXIST + raise DestinationExistsError(e.filename,opname=opname,details=e) + if e.errno == errno.ENOTDIR: + raise ResourceInvalidError(e.filename,opname=opname,details=e) + if e.errno == errno.EISDIR: + raise ResourceInvalidError(e.filename,opname=opname,details=e) + if e.errno == errno.EINVAL: + raise ResourceInvalidError(e.filename,opname=opname,details=e) + raise OperationFailedError(opname,details=e) + return wrapper + + diff --git a/fs/expose/fuse/__init__.py b/fs/expose/fuse/__init__.py index f21a3a1..dbf25ff 100644 --- a/fs/expose/fuse/__init__.py +++ b/fs/expose/fuse/__init__.py @@ -78,15 +78,10 @@ def handle_fs_errors(func): equivalent OSError. It also makes the function return zero instead of None as an indication of successful execution. """ + func = convert_fs_errors(func) + @wraps(func) def wrapper(*args,**kwds): - try: - res = func(*args,**kwds) - except ResourceNotFoundError, e: - raise OSError(errno.ENOENT,str(e)) - except FSError, e: - raise OSError(errno.EFAULT,str(e)) - except Exception, e: - raise + res = func(*args,**kwds) if res is None: return 0 return res diff --git a/fs/expose/sftp.py b/fs/expose/sftp.py index 526fd84..0cd3682 100644 --- a/fs/expose/sftp.py +++ b/fs/expose/sftp.py @@ -36,11 +36,7 @@ from fs.path import * from fs.errors import * -try: - from functools import wraps -except ImportError: - def wraps(f): - return f +from fs.errors import wraps # Default host key used by BaseSFTPServer @@ -137,7 +133,7 @@ class SFTPServerInterface(paramiko.SFTPServerInterface): return paramiko.SFTP_OK def canonicalize(self,path): - return abspath(path) + return abspath(normpath(path)) def chattr(self,path,attr): return paramiko.SFTP_OP_UNSUPPORTED diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 2fc4a86..278f906 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -185,326 +185,274 @@ class MemoryFS(FS): def __unicode__(self):
return unicode(self.__str__())
+ @synchronize
def _get_dir_entry(self, dirpath):
- self._lock.acquire()
- try:
- current_dir = self.root
- for path_component in iteratepath(dirpath):
- if current_dir.contents is None:
- return None
- dir_entry = current_dir.contents.get(path_component, None)
- if dir_entry is None:
- return None
- current_dir = dir_entry
-
- return current_dir
- finally:
- self._lock.release()
+ current_dir = self.root
+ for path_component in iteratepath(dirpath):
+ if current_dir.contents is None:
+ return None
+ dir_entry = current_dir.contents.get(path_component, None)
+ if dir_entry is None:
+ return None
+ current_dir = dir_entry
+ return current_dir
+ @synchronize
def desc(self, path):
- self._lock.acquire()
- try:
- if self.isdir(path):
- return "Memory dir"
- elif self.isfile(path):
- return "Memory file object"
- else:
- return "No description available"
- finally:
- self._lock.release()
+ if self.isdir(path):
+ return "Memory dir"
+ elif self.isfile(path):
+ return "Memory file object"
+ else:
+ return "No description available"
+ @synchronize
def isdir(self, path):
- self._lock.acquire()
- try:
- dir_item = self._get_dir_entry(normpath(path))
- if dir_item is None:
- return False
- return dir_item.isdir()
- finally:
- self._lock.release()
+ dir_item = self._get_dir_entry(normpath(path))
+ if dir_item is None:
+ return False
+ return dir_item.isdir()
+ @synchronize
def isfile(self, path):
- self._lock.acquire()
- try:
- dir_item = self._get_dir_entry(normpath(path))
- if dir_item is None:
- return False
- return dir_item.isfile()
- finally:
- self._lock.release()
+ dir_item = self._get_dir_entry(normpath(path))
+ if dir_item is None:
+ return False
+ return dir_item.isfile()
+ @synchronize
def exists(self, path):
- self._lock.acquire()
- try:
- return self._get_dir_entry(path) is not None
- finally:
- self._lock.release()
+ return self._get_dir_entry(path) is not None
- def makedir(self, dirname, mode=0777, recursive=False, allow_recreate=False):
+ @synchronize
+ def makedir(self, dirname, recursive=False, allow_recreate=False):
if not dirname:
raise PathError("", "Path is empty")
- self._lock.acquire()
- try:
- fullpath = dirname
- dirpath, dirname = pathsplit(dirname)
-
- if recursive:
- parent_dir = self._get_dir_entry(dirpath)
- if parent_dir is not None:
- if parent_dir.isfile():
- raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
- else:
- if not allow_recreate:
- if dirname in parent_dir.contents:
- raise DestinationExistsError(dirname, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")
-
- current_dir = self.root
- for path_component in iteratepath(dirpath)[:-1]:
- dir_item = current_dir.contents.get(path_component, None)
- if dir_item is None:
- break
- if not dir_item.isdir():
- raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
- current_dir = dir_item
+ fullpath = dirname
+ dirpath, dirname = pathsplit(dirname)
- current_dir = self.root
- for path_component in iteratepath(dirpath):
- dir_item = current_dir.contents.get(path_component, None)
- if dir_item is None:
- new_dir = self._make_dir_entry("dir", path_component)
- current_dir.contents[path_component] = new_dir
- current_dir = new_dir
- else:
- current_dir = dir_item
-
- parent_dir = current_dir
+ if recursive:
+ parent_dir = self._get_dir_entry(dirpath)
+ if parent_dir is not None:
+ if parent_dir.isfile():
+ raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
+ else:
+ if not allow_recreate:
+ if dirname in parent_dir.contents:
+ raise DestinationExistsError(dirname, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")
- else:
- parent_dir = self._get_dir_entry(dirpath)
- if parent_dir is None:
- raise ParentDirectoryMissingError(dirname, msg="Could not make dir, as parent dir does not exist: %(path)s")
+ current_dir = self.root
+ for path_component in iteratepath(dirpath)[:-1]:
+ dir_item = current_dir.contents.get(path_component, None)
+ if dir_item is None:
+ break
+ if not dir_item.isdir():
+ raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
+ current_dir = dir_item
- dir_item = parent_dir.contents.get(dirname, None)
- if dir_item is not None:
- if dir_item.isdir():
- if not allow_recreate:
- raise DestinationExistsError(dirname)
+ current_dir = self.root
+ for path_component in iteratepath(dirpath):
+ dir_item = current_dir.contents.get(path_component, None)
+ if dir_item is None:
+ new_dir = self._make_dir_entry("dir", path_component)
+ current_dir.contents[path_component] = new_dir
+ current_dir = new_dir
else:
- raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
+ current_dir = dir_item
- if dir_item is None:
- parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)
+ parent_dir = current_dir
- return self
- finally:
- self._lock.release()
+ else:
+ parent_dir = self._get_dir_entry(dirpath)
+ if parent_dir is None:
+ raise ParentDirectoryMissingError(dirname, msg="Could not make dir, as parent dir does not exist: %(path)s")
+
+ dir_item = parent_dir.contents.get(dirname, None)
+ if dir_item is not None:
+ if dir_item.isdir():
+ if not allow_recreate:
+ raise DestinationExistsError(dirname)
+ else:
+ raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
+
+ if dir_item is None:
+ parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)
+
+ return self
def _orphan_files(self, file_dir_entry):
for f in file_dir_entry.open_files:
f.close()
+ @synchronize
def _lock_dir_entry(self, path):
- self._lock.acquire()
- try:
- dir_entry = self._get_dir_entry(path)
- dir_entry.lock()
- finally:
- self._lock.release()
+ dir_entry = self._get_dir_entry(path)
+ dir_entry.lock()
+ @synchronize
def _unlock_dir_entry(self, path):
- self._lock.acquire()
- try:
- dir_entry = self._get_dir_entry(path)
- dir_entry.unlock()
- finally:
- self._lock.release()
+ dir_entry = self._get_dir_entry(path)
+ dir_entry.unlock()
+ @synchronize
def _is_dir_locked(self, path):
- self._lock.acquire()
- try:
- dir_entry = self._get_dir_entry(path)
- return dir_entry.islocked()
- finally:
- self._lock.release()
+ dir_entry = self._get_dir_entry(path)
+ return dir_entry.islocked()
+ @synchronize
def open(self, path, mode="r", **kwargs):
- self._lock.acquire()
- try:
- filepath, filename = pathsplit(path)
- parent_dir_entry = self._get_dir_entry(filepath)
+ filepath, filename = pathsplit(path)
+ parent_dir_entry = self._get_dir_entry(filepath)
- if parent_dir_entry is None or not parent_dir_entry.isdir():
- raise FileNotFoundError(path)
+ if parent_dir_entry is None or not parent_dir_entry.isdir():
+ raise ResourceNotFoundError(path)
- if 'r' in mode or 'a' in mode:
- if filename not in parent_dir_entry.contents:
- raise FileNotFoundError(path)
+ if 'r' in mode or 'a' in mode:
+ if filename not in parent_dir_entry.contents:
+ raise ResourceNotFoundError(path)
- file_dir_entry = parent_dir_entry.contents[filename]
+ file_dir_entry = parent_dir_entry.contents[filename]
- if 'a' in mode and file_dir_entry.islocked():
- raise ResourceLockedError(path)
+ if 'a' in mode and file_dir_entry.islocked():
+ raise ResourceLockedError(path)
- self._lock_dir_entry(path)
- mem_file = self.file_factory(path, self, file_dir_entry.data, mode)
- file_dir_entry.open_files.append(mem_file)
- return mem_file
+ self._lock_dir_entry(path)
+ mem_file = self.file_factory(path, self, file_dir_entry.data, mode)
+ file_dir_entry.open_files.append(mem_file)
+ return mem_file
- elif 'w' in mode:
- if filename not in parent_dir_entry.contents:
- file_dir_entry = self._make_dir_entry("file", filename)
- parent_dir_entry.contents[filename] = file_dir_entry
- else:
- file_dir_entry = parent_dir_entry.contents[filename]
+ elif 'w' in mode:
+ if filename not in parent_dir_entry.contents:
+ file_dir_entry = self._make_dir_entry("file", filename)
+ parent_dir_entry.contents[filename] = file_dir_entry
+ else:
+ file_dir_entry = parent_dir_entry.contents[filename]
- if file_dir_entry.islocked():
- raise ResourceLockedError(path)
+ if file_dir_entry.islocked():
+ raise ResourceLockedError(path)
- self._lock_dir_entry(path)
+ self._lock_dir_entry(path)
- mem_file = self.file_factory(path, self, None, mode)
- file_dir_entry.open_files.append(mem_file)
- return mem_file
+ mem_file = self.file_factory(path, self, None, mode)
+ file_dir_entry.open_files.append(mem_file)
+ return mem_file
- if parent_dir_entry is None:
- raise FileNotFoundError(path)
- finally:
- self._lock.release()
+ if parent_dir_entry is None:
+ raise ResourceNotFoundError(path)
+ @synchronize
def remove(self, path):
- self._lock.acquire()
- try:
- dir_entry = self._get_dir_entry(path)
+ dir_entry = self._get_dir_entry(path)
- if dir_entry is None:
- raise FileNotFoundError(path)
+ if dir_entry is None:
+ raise ResourceNotFoundError(path)
- if dir_entry.islocked():
- self._orphan_files(dir_entry)
- #raise ResourceLockedError("FILE_LOCKED", path)
+ if dir_entry.islocked():
+ self._orphan_files(dir_entry)
+ #raise ResourceLockedError("FILE_LOCKED", path)
- if dir_entry.isdir():
- raise ResourceInvalidError(path,msg="That's a directory, not a file: %(path)s")
+ if dir_entry.isdir():
+ raise ResourceInvalidError(path,msg="That's a directory, not a file: %(path)s")
- pathname, dirname = pathsplit(path)
+ pathname, dirname = pathsplit(path)
- parent_dir = self._get_dir_entry(pathname)
+ parent_dir = self._get_dir_entry(pathname)
- del parent_dir.contents[dirname]
- finally:
- self._lock.release()
+ del parent_dir.contents[dirname]
+ @synchronize
def removedir(self, path, recursive=False, force=False):
- self._lock.acquire()
- try:
- dir_entry = self._get_dir_entry(path)
-
- if dir_entry is None:
- raise DirectoryNotFoundError(path)
- if dir_entry.islocked():
- raise ResourceLockedError(path)
- if not dir_entry.isdir():
- raise ResourceInvalidError(path, msg="Can't remove resource, its not a directory: %(path)s" )
-
- if dir_entry.contents and not force:
- raise DirectoryNotEmptyError(path)
-
- if recursive:
- rpathname = path
- while rpathname:
- rpathname, dirname = pathsplit(rpathname)
- parent_dir = self._get_dir_entry(rpathname)
- del parent_dir.contents[dirname]
- else:
- pathname, dirname = pathsplit(path)
- parent_dir = self._get_dir_entry(pathname)
+ dir_entry = self._get_dir_entry(path)
+
+ if dir_entry is None:
+ raise ResourceNotFoundError(path)
+ if dir_entry.islocked():
+ raise ResourceLockedError(path)
+ if not dir_entry.isdir():
+ raise ResourceInvalidError(path, msg="Can't remove resource, its not a directory: %(path)s" )
+
+ if dir_entry.contents and not force:
+ raise DirectoryNotEmptyError(path)
+
+ if recursive:
+ rpathname = path
+ while rpathname:
+ rpathname, dirname = pathsplit(rpathname)
+ parent_dir = self._get_dir_entry(rpathname)
del parent_dir.contents[dirname]
+ else:
+ pathname, dirname = pathsplit(path)
+ parent_dir = self._get_dir_entry(pathname)
+ del parent_dir.contents[dirname]
- finally:
- self._lock.release()
+ @synchronize
def rename(self, src, dst):
if not issamedir(src, dst):
raise ValueError("Destination path must the same directory (use the move method for moving to a different directory)")
- self._lock.acquire()
- try:
- dst = pathsplit(dst)[-1]
+ dst = pathsplit(dst)[-1]
- dir_entry = self._get_dir_entry(src)
- if dir_entry is None:
- raise DirectoryNotFoundError(src)
- #if dir_entry.islocked():
- # raise ResourceLockedError("FILE_LOCKED", src)
+ dir_entry = self._get_dir_entry(src)
+ if dir_entry is None:
+ raise ResourceNotFoundError(src)
+ #if dir_entry.islocked():
+ # raise ResourceLockedError("FILE_LOCKED", src)
- open_files = dir_entry.open_files[:]
- for f in open_files:
- f.flush()
- f.path = dst
+ open_files = dir_entry.open_files[:]
+ for f in open_files:
+ f.flush()
+ f.path = dst
- dst_dir_entry = self._get_dir_entry(dst)
- if dst_dir_entry is not None:
- raise DestinationExistsError(path)
+ dst_dir_entry = self._get_dir_entry(dst)
+ if dst_dir_entry is not None:
+ raise DestinationExistsError(path)
- pathname, dirname = pathsplit(src)
- parent_dir = self._get_dir_entry(pathname)
- parent_dir.contents[dst] = parent_dir.contents[dirname]
- parent_dir.name = dst
- del parent_dir.contents[dirname]
+ pathname, dirname = pathsplit(src)
+ parent_dir = self._get_dir_entry(pathname)
+ parent_dir.contents[dst] = parent_dir.contents[dirname]
+ parent_dir.name = dst
+ del parent_dir.contents[dirname]
- finally:
- self._lock.release()
+ @synchronize
def _on_close_memory_file(self, open_file, path, value):
- self._lock.acquire()
- try:
- filepath, filename = pathsplit(path)
- dir_entry = self._get_dir_entry(path)
- if dir_entry is not None and value is not None:
- dir_entry.data = value
- dir_entry.open_files.remove(open_file)
- self._unlock_dir_entry(path)
- finally:
- self._lock.release()
+ filepath, filename = pathsplit(path)
+ dir_entry = self._get_dir_entry(path)
+ if dir_entry is not None and value is not None:
+ dir_entry.data = value
+ dir_entry.open_files.remove(open_file)
+ self._unlock_dir_entry(path)
+ @synchronize
def _on_flush_memory_file(self, path, value):
- self._lock.acquire()
- try:
- filepath, filename = pathsplit(path)
- dir_entry = self._get_dir_entry(path)
- dir_entry.data = value
- finally:
- self._lock.release()
+ filepath, filename = pathsplit(path)
+ dir_entry = self._get_dir_entry(path)
+ dir_entry.data = value
+ @synchronize
def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
- self._lock.acquire()
- try:
- dir_entry = self._get_dir_entry(path)
- if dir_entry is None:
- raise DirectoryNotFoundError(path)
- if dir_entry.isfile():
- raise ResourceInvalidError(path,msg="that's a file, not a directory: %(path)s")
- paths = dir_entry.contents.keys()
- return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only)
- finally:
- self._lock.release()
-
+ dir_entry = self._get_dir_entry(path)
+ if dir_entry is None:
+ raise ResourceNotFoundError(path)
+ if dir_entry.isfile():
+ raise ResourceInvalidError(path,msg="that's a file, not a directory: %(path)s")
+ paths = dir_entry.contents.keys()
+ return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only)
+
+ @synchronize
def getinfo(self, path):
- self._lock.acquire()
- try:
- dir_entry = self._get_dir_entry(path)
+ dir_entry = self._get_dir_entry(path)
- if dir_entry is None:
- raise ResourceNotFoundError(path)
+ if dir_entry is None:
+ raise ResourceNotFoundError(path)
- info = {}
- info['created_time'] = dir_entry.created_time
+ info = {}
+ info['created_time'] = dir_entry.created_time
- if dir_entry.isfile():
- info['size'] = len(dir_entry.data or '')
+ if dir_entry.isfile():
+ info['size'] = len(dir_entry.data or '')
- return info
- finally:
- self._lock.release()
+ return info
diff --git a/fs/mountfs.py b/fs/mountfs.py index 2bef74d..96913af 100644 --- a/fs/mountfs.py +++ b/fs/mountfs.py @@ -105,7 +105,7 @@ class MountFS(FS): fs, mount_path, delegate_path = self._delegate(path) if fs is None: - raise DirectoryNotFoundError(path) + raise ResourceNotFoundError(path) if fs is self: if files_only: @@ -128,9 +128,9 @@ class MountFS(FS): files_only=files_only) if full or absolute: if full: - path = abspath(path) + path = abspath(normpath(path)) else: - path = relpath(path) + path = relpath(normpath(path)) paths = [pathjoin(path, p) for p in paths] return paths @@ -161,7 +161,7 @@ class MountFS(FS): fs, mount_path, delegate_path = self._delegate(path) if fs is None: - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) return fs.open(delegate_path, mode, **kwargs) @@ -193,7 +193,7 @@ class MountFS(FS): path = normpath(path) fs, mount_path, delegate_path = self._delegate(path) if fs is None: - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) if fs is self: raise UnsupportedError("remove file", msg="Can only remove paths within a mounted dir") return fs.remove(delegate_path) @@ -300,13 +300,13 @@ class MountFS(FS): fs, mount_path, delegate_path = self._delegate(path) if fs is None: - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) if fs is self: object = self.mount_tree.get(path, None) if object is None or isinstance(object, dict): - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) size = self.mount_tree[path].info_callable(path).get("size", None) return size diff --git a/fs/multifs.py b/fs/multifs.py index 0e1e357..3454e64 100644 --- a/fs/multifs.py +++ b/fs/multifs.py @@ -136,7 +136,7 @@ class MultiFS(FS): fs_file = fs.open(path, mode, **kwargs) return fs_file - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) finally: self._lock.release() @@ -188,7 +188,7 @@ class MultiFS(FS): if fs.exists(path): fs.remove(path) return - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) finally: self._lock.release() @@ -199,7 +199,7 @@ class MultiFS(FS): if fs.isdir(path): fs.removedir(path, recursive) return - raise DirectoryNotFoundError(path) + raise ResourceNotFoundError(path) finally: self._lock.release() @@ -21,13 +21,11 @@ class OSFS(FS): def __init__(self, root_path, dir_mode=0700, thread_synchronize=True): FS.__init__(self, thread_synchronize=thread_synchronize) - expanded_path = normpath(os.path.abspath(os.path.expanduser(os.path.expandvars(root_path)))) if not os.path.exists(expanded_path): - raise DirectoryNotFoundError(expanded_path, msg="Root directory does not exist: %(path)s") + raise ResourceNotFoundError(expanded_path,msg="Root directory does not exist: %(path)s") if not os.path.isdir(expanded_path): - raise InvalidResourceError(expanded_path, msg="Root path is not a directory: %(path)s") - + raise ResourceInvalidError(expanded_path,msg="Root path is not a directory: %(path)s") self.root_path = normpath(os.path.abspath(expanded_path)) self.dir_mode = dir_mode @@ -38,40 +36,32 @@ class OSFS(FS): sys_path = os.path.join(self.root_path, relpath(path)).replace('/', os.sep) return sys_path + @convert_os_errors def open(self, path, mode="r", **kwargs): mode = filter(lambda c: c in "rwabt+",mode) - try: - f = open(self.getsyspath(path), mode, kwargs.get("buffering", -1)) - except IOError, e: - if e.errno == 2: - raise FileNotFoundError(path) - raise OperationFailedError("open file", details=e, msg=str(e)) - - return f + return open(self.getsyspath(path), mode, kwargs.get("buffering", -1)) + @convert_os_errors def exists(self, path): path = self.getsyspath(path) return os.path.exists(path) + @convert_os_errors def isdir(self, path): path = self.getsyspath(path) return os.path.isdir(path) + @convert_os_errors def isfile(self, path): path = self.getsyspath(path) return os.path.isfile(path) + @convert_os_errors def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): - try: - paths = os.listdir(self.getsyspath(path)) - except (OSError, IOError), e: - if e.errno == 2: - raise ResourceNotFoundError(path) - if e.errno in (20,22,): - raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s") - raise OperationFailedError("list directory", path=path, details=e, msg="Unable to get directory listing: %(path)s - (%(details)s)") + paths = os.listdir(self.getsyspath(path)) return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only) + @convert_os_errors def makedir(self, path, recursive=False, allow_recreate=False): sys_path = self.getsyspath(path) try: @@ -87,22 +77,15 @@ class OSFS(FS): raise DestinationExistsError(path,msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s") elif e.errno == 2: raise ParentDirectoryMissingError(path) - elif e.errno == 22: - raise ResourceInvalidError(path) else: - raise OperationFailedError("make directory",path=path,details=e) + raise + @convert_os_errors def remove(self, path): sys_path = self.getsyspath(path) - try: - os.remove(sys_path) - except OSError, e: - if not self.exists(path): - raise ResourceNotFoundError(path) - if self.isdir(path): - raise ResourceInvalidError(path,msg="Cannot use remove() on a directory: %(path)s") - raise OperationFailedError("remove file", path=path, details=e) + os.remove(sys_path) + @convert_os_errors def removedir(self, path, recursive=False,force=False): sys_path = self.getsyspath(path) # Don't remove the root directory of this FS @@ -113,14 +96,7 @@ class OSFS(FS): self.remove(path2) for path2 in self.listdir(path,absolute=True,dirs_only=True): self.removedir(path2,force=True) - try: - os.rmdir(sys_path) - except OSError, e: - if self.isfile(path): - raise ResourceInvalidError(path,msg="Can't use removedir() on a file: %(path)s") - if self.listdir(path): - raise DirectoryNotEmptyError(path) - raise OperationFailedError("remove directory", path=path, details=e) + os.rmdir(sys_path) # Using os.removedirs() for this can result in dirs being # removed outside the root of this FS, so we recurse manually. if recursive: @@ -129,26 +105,21 @@ class OSFS(FS): except DirectoryNotEmptyError: pass + @convert_os_errors def rename(self, src, dst): if not issamedir(src, dst): raise ValueError("Destination path must the same directory (use the move method for moving to a different directory)") path_src = self.getsyspath(src) path_dst = self.getsyspath(dst) - try: - os.rename(path_src, path_dst) - except OSError, e: - raise OperationFailedError("rename resource", path=src, details=e) + os.rename(path_src, path_dst) + @convert_os_errors def getinfo(self, path): sys_path = self.getsyspath(path) - try: - stats = os.stat(sys_path) - except OSError, e: - if e.errno == 2: - raise ResourceNotFoundError(path) - raise ResourceError(path, details=e) + stats = os.stat(sys_path) info = dict((k, getattr(stats, k)) for k in dir(stats) if not k.startswith('__') ) info['size'] = info['st_size'] + # TODO: this doesn't actually mean 'creation time' on unix ct = info.get('st_ctime', None) if ct is not None: info['created_time'] = datetime.datetime.fromtimestamp(ct) @@ -160,43 +131,35 @@ class OSFS(FS): info['modified_time'] = datetime.datetime.fromtimestamp(at) return info - + @convert_os_errors def getsize(self, path): sys_path = self.getsyspath(path) - try: - stats = os.stat(sys_path) - except OSError, e: - raise ResourceError(path, details=e) + stats = os.stat(sys_path) return stats.st_size # Provide native xattr support if available if xattr: + @convert_os_errors def setxattr(self, path, key, value): - try: - xattr.xattr(self.getsyspath(path))[key]=value - except IOError, e: - raise OperationFailedError('set extended attribute', path=path, details=e) + xattr.xattr(self.getsyspath(path))[key]=value + @convert_os_errors def getxattr(self, path, key, default=None): try: return xattr.xattr(self.getsyspath(path)).get(key) except KeyError: return default - except IOError, e: - raise OperationFailedError('get extended attribute', path=path, details=e) + @convert_os_errors def delxattr(self, path, key): try: del xattr.xattr(self.getsyspath(path))[key] except KeyError: pass - except IOError, e: - raise OperationFailedError('delete extended attribute', path=path, details=e) + @convert_os_errors def listxattrs(self, path): - try: - return xattr.xattr(self.getsyspath(path)).keys() - except IOError, e: - raise OperationFailedError('list extended attributes', path=path, details=e) + return xattr.xattr(self.getsyspath(path)).keys() + @@ -62,13 +62,12 @@ def iteratepath(path, numsplits=None): def abspath(path): - """Convert the given path to a normalized, absolute path. + """Convert the given path to an absolute path. Since FS objects have no concept of a 'current directory' this simply adds a leading '/' character if the path doesn't already have one. """ - path = normpath(path) if not path: return "/" if path[0] != "/": @@ -77,13 +76,12 @@ def abspath(path): def relpath(path): - """Convert the given path to a normalized, relative path. + """Convert the given path to a relative path. This is the inverse of abspath(), stripping a leading '/' from the path if it is present. """ - path = normpath(path) while path and path[0] == "/": path = path[1:] return path diff --git a/fs/rpcfs.py b/fs/rpcfs.py index d53d082..f5b4970 100644 --- a/fs/rpcfs.py +++ b/fs/rpcfs.py @@ -141,7 +141,7 @@ class RPCFS(FS): data = self.proxy.get_contents(path).data except IOError: if "w" not in mode and "a" not in mode: - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) if not self.isdir(dirname(path)): raise ParentDirectoryMissingError(path) self.proxy.set_contents(path,xmlrpclib.Binary("")) @@ -172,7 +172,7 @@ class S3FS(FS): def _s3path(self,path): """Get the absolute path to a file stored in S3.""" - path = relpath(path) + path = relpath(normpath(path)) path = self._separator.join(iteratepath(path)) s3path = self._prefix + path if s3path and s3path[-1] == self._separator: @@ -235,7 +235,7 @@ class S3FS(FS): if k is None: # Create the file if it's missing if "w" not in mode and "a" not in mode: - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) if not self.isdir(dirname(path)): raise ParentDirectoryMissingError(path) k = self._sync_set_contents(s3path,"") @@ -311,7 +311,7 @@ class S3FS(FS): if s3path != self._prefix: if self.isfile(path): raise ResourceInvalidError(path,msg="that's not a directory: %(path)s") - raise DirectoryNotFoundError(path) + raise ResourceNotFoundError(path) return self._listdir_helper(path,paths,wildcard,full,absolute,dirs_only,files_only) def _listdir_helper(self,path,paths,wildcard,full,absolute,dirs_only,files_only): @@ -391,7 +391,7 @@ class S3FS(FS): if k.name.startswith(s3path + "/"): raise ResourceInvalidError(path,msg="that's not a file: %(path)s") else: - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) self._s3bukt.delete_key(s3path) k = self._s3bukt.get_key(s3path) while k: @@ -419,7 +419,7 @@ class S3FS(FS): if not found: if self.isfile(path): raise ResourceInvalidError(path,msg="removedir() called on a regular file: %(path)s") - raise DirectoryNotFoundError(path) + raise ResourceNotFoundError(path) self._s3bukt.delete_key(s3path) if recursive and path not in ("","/"): pdir = dirname(path) diff --git a/fs/sftpfs.py b/fs/sftpfs.py index f1eeac5..1d825b6 100644 --- a/fs/sftpfs.py +++ b/fs/sftpfs.py @@ -52,7 +52,7 @@ class SFTPFS(FS): if not connection.is_authenticated(): connection.connect(**credentials) self.client = paramiko.SFTPClient.from_transport(connection) - self.root = abspath(root) + self.root = abspath(normpath(root)) def __del__(self): self.close() @@ -83,24 +83,21 @@ class SFTPFS(FS): self.client = None def _normpath(self,path): - npath = pathjoin(self.root,relpath(path)) + npath = pathjoin(self.root,relpath(normpath(path))) if not isprefix(self.root,npath): raise PathError(path,msg="Path is outside root: %(path)s") return npath + @convert_os_errors def open(self,path,mode="r",bufsize=-1): npath = self._normpath(path) - try: - f = self.client.open(npath,mode,bufsize) - except IOError, e: - if getattr(e,"errno",None) == 2: - raise FileNotFoundError(path) - raise OperationFailedError("open file",path=path,details=e) + f = self.client.open(npath,mode,bufsize) if self.isdir(path): msg = "that's a directory: %(path)s" raise ResourceInvalidError(path,msg=msg) return f + @convert_os_errors def exists(self,path): npath = self._normpath(path) try: @@ -108,10 +105,10 @@ class SFTPFS(FS): except IOError, e: if getattr(e,"errno",None) == 2: return False - raise OperationFailedError("exists",path,details=e) - else: - return True + raise + return True + @convert_os_errors def isdir(self,path): npath = self._normpath(path) try: @@ -119,9 +116,10 @@ class SFTPFS(FS): except IOError, e: if getattr(e,"errno",None) == 2: return False - raise OperationFailedError("isdir",path,details=e) + raise return statinfo.S_ISDIR(stat.st_mode) + @convert_os_errors def isfile(self,path): npath = self._normpath(path) try: @@ -129,9 +127,10 @@ class SFTPFS(FS): except IOError, e: if getattr(e,"errno",None) == 2: return False - raise OperationFailedError("isfile",path,details=e) + raise return statinfo.S_ISREG(stat.st_mode) + @convert_os_errors def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False): npath = self._normpath(path) try: @@ -143,9 +142,10 @@ class SFTPFS(FS): raise ResourceNotFoundError(path) elif self.isfile(path): raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s") - raise OperationFailedError("list directory", path=path, details=e, msg="Unable to get directory listing: %(path)s - (%(details)s)") + raise return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only) + @convert_os_errors def makedir(self,path,recursive=False,allow_recreate=False): npath = self._normpath(path) try: @@ -162,8 +162,8 @@ class SFTPFS(FS): self.makedir(dirname(path),recursive=True) self.makedir(path,allow_recreate=allow_recreate) else: - # Undetermined error - raise OperationFailedError("make directory",path=path,details=e) + # Undetermined error, let the decorator handle it + raise else: # Destination exists if statinfo.S_ISDIR(stat.st_mode): @@ -172,17 +172,19 @@ class SFTPFS(FS): else: raise ResourceInvalidError(path,msg="Can't create directory, there's already a file of that name: %(path)s") + @convert_os_errors def remove(self,path): npath = self._normpath(path) try: self.client.remove(npath) except IOError, e: if getattr(e,"errno",None) == 2: - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) elif self.isdir(path): raise ResourceInvalidError(path,msg="Cannot use remove() on a directory: %(path)s") - raise OperationFailedError("remove file", path=path, details=e) + raise + @convert_os_errors def removedir(self,path,recursive=False,force=False): npath = self._normpath(path) if path in ("","/"): @@ -199,16 +201,17 @@ class SFTPFS(FS): if getattr(e,"errno",None) == 2: if self.isfile(path): raise ResourceInvalidError(path,msg="Can't use removedir() on a file: %(path)s") - raise DirectoryNotFoundError(path) + raise ResourceNotFoundError(path) elif self.listdir(path): raise DirectoryNotEmptyError(path) - raise OperationFailedError("remove directory", path=path, details=e) + raise if recursive: try: self.removedir(dirname(path),recursive=True) except DirectoryNotEmptyError: pass + @convert_os_errors def rename(self,src,dst): if not issamedir(src, dst): raise ValueError("Destination path must the same directory (use the move method for moving to a different directory)") @@ -218,9 +221,10 @@ class SFTPFS(FS): self.client.rename(nsrc,ndst) except IOError, e: if getattr(e,"errno",None) == 2: - raise FileNotFoundError(path) - raise OperationFailedError("rename resource", path=src, details=e) + raise ResourceNotFoundError(path) + raise + @convert_os_errors def move(self,src,dst,overwrite=False,chunk_size=16384): nsrc = self._normpath(src) ndst = self._normpath(dst) @@ -230,13 +234,14 @@ class SFTPFS(FS): self.client.rename(nsrc,ndst) except IOError, e: if getattr(e,"errno",None) == 2: - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) if self.exists(dst): raise DestinationExistsError(dst) if not self.isdir(dirname(dst)): raise ParentDirectoryMissingError(dst,msg="Destination directory does not exist: %(path)s") - raise OperationFailedError("move file", path=src, details=e) + raise + @convert_os_errors def movedir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384): nsrc = self._normpath(src) ndst = self._normpath(dst) @@ -246,19 +251,17 @@ class SFTPFS(FS): self.client.rename(nsrc,ndst) except IOError, e: if getattr(e,"errno",None) == 2: - raise DirNotFoundError(path) + raise ResourceNotFoundError(path) if self.exists(dst): raise DestinationExistsError(dst) if not self.isdir(dirname(dst)): raise ParentDirectoryMissingError(dst,msg="Destination directory does not exist: %(path)s") - raise OperationFailedError("move directory", path=src, details=e) + raise + @convert_os_errors def getinfo(self, path): npath = self._normpath(path) - try: - stats = self.client.stat(npath) - except IOError, e: - raise ResourceError(path, details=e) + stats = self.client.stat(npath) info = dict((k, getattr(stats, k)) for k in dir(stats) if not k.startswith('__') ) info['size'] = info['st_size'] ct = info.get('st_ctime', None) @@ -272,12 +275,10 @@ class SFTPFS(FS): info['modified_time'] = datetime.datetime.fromtimestamp(at) return info + @convert_os_errors def getsize(self, path): npath = self._normpath(path) - try: - stats = self.client.stat(npath) - except OSError, e: - raise ResourceError(path, details=e) + stats = self.client.stat(npath) return stats.st_size diff --git a/fs/tests/__init__.py b/fs/tests/__init__.py index a2fc050..bcdc6dd 100644 --- a/fs/tests/__init__.py +++ b/fs/tests/__init__.py @@ -447,6 +447,4 @@ class FSTestCases: self.fs.createfile("test1","hello world") fs2 = pickle.loads(pickle.dumps(self.fs)) self.assert_(fs2.isfile("test1")) - if hasattr(fs2,"close"): - fs2.close() diff --git a/fs/tests/test_s3fs.py b/fs/tests/test_s3fs.py index 74a4e9a..13531b9 100644 --- a/fs/tests/test_s3fs.py +++ b/fs/tests/test_s3fs.py @@ -18,7 +18,7 @@ from fs import s3fs class TestS3FS(unittest.TestCase,FSTestCases): # Disable the tests by default - __test__ = False + #__test__ = False bucket = "test-s3fs.rfk.id.au" diff --git a/fs/zipfs.py b/fs/zipfs.py index 3da4da4..2d1bdec 100644 --- a/fs/zipfs.py +++ b/fs/zipfs.py @@ -74,7 +74,7 @@ class ZipFS(FS): try: self.zf = ZipFile(zip_file, mode, compression_type, allowZip64) except IOError: - raise FileNotFoundError(str(zip_file), msg="Zip file does not exist: %(path)s") + raise ResourceNotFoundError(str(zip_file), msg="Zip file does not exist: %(path)s") self.zip_path = str(zip_file) self.temp_fs = None @@ -132,7 +132,7 @@ class ZipFS(FS): try: contents = self.zf.read(path) except KeyError: - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) return StringIO(contents) if 'w' in mode: @@ -153,12 +153,12 @@ class ZipFS(FS): self._lock.acquire() try: if not self.exists(path): - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) path = normpath(path) try: contents = self.zf.read(path) except KeyError: - raise FileNotFoundError(path) + raise ResourceNotFoundError(path) except RuntimeError: raise OperationFailedError("read file", path=path, msg="Zip file must be oppened with 'r' or 'a' to read") return contents |