diff options
author | willmcgugan <willmcgugan@67cdc799-7952-0410-af00-57a81ceafa0f> | 2010-12-09 22:09:48 +0000 |
---|---|---|
committer | willmcgugan <willmcgugan@67cdc799-7952-0410-af00-57a81ceafa0f> | 2010-12-09 22:09:48 +0000 |
commit | 6c672de2810f4e9195c31c7f3e0a0313af60206b (patch) | |
tree | 33ed76811fffca8e0014842c5c4b853efc48c0ba /fs | |
parent | 6b1c5e2801216e185f2e781d4b2f6bd5b12fd020 (diff) | |
download | pyfilesystem-6c672de2810f4e9195c31c7f3e0a0313af60206b.tar.gz |
Changed syntax for commands to be more url like, optimized sftps to use fewer queries for listdir
git-svn-id: http://pyfilesystem.googlecode.com/svn/trunk@543 67cdc799-7952-0410-af00-57a81ceafa0f
Diffstat (limited to 'fs')
-rw-r--r-- | fs/commands/fscp.py | 4 | ||||
-rw-r--r-- | fs/commands/fsls.py | 6 | ||||
-rw-r--r-- | fs/commands/fstree.py | 7 | ||||
-rw-r--r-- | fs/commands/runner.py | 15 | ||||
-rw-r--r-- | fs/ftpfs.py | 10 | ||||
-rw-r--r-- | fs/memoryfs.py | 33 | ||||
-rw-r--r-- | fs/opener.py | 214 | ||||
-rw-r--r-- | fs/path.py | 13 | ||||
-rw-r--r-- | fs/sftpfs.py | 166 | ||||
-rw-r--r-- | fs/utils.py | 34 |
10 files changed, 381 insertions, 121 deletions
diff --git a/fs/commands/fscp.py b/fs/commands/fscp.py index 81915f1..1e30bc3 100644 --- a/fs/commands/fscp.py +++ b/fs/commands/fscp.py @@ -1,6 +1,6 @@ from fs.opener import opener from fs.utils import copyfile, copystructure -from fs.path import pathjoin +from fs.path import pathjoin, iswildcard from fs.errors import FSError from fs.commands.runner import Command import sys @@ -92,7 +92,7 @@ Copy SOURCE to DESTINATION""" if src_path is None: src_path = '/' - if self.is_wildcard(src_path): + if iswildcard(src_path): for file_path in src_fs.listdir(wildcard=src_path, full=True): copy_fs_paths.append((self.FILE, src_fs, file_path, file_path)) diff --git a/fs/commands/fsls.py b/fs/commands/fsls.py index 3504c14..abfc6b8 100644 --- a/fs/commands/fsls.py +++ b/fs/commands/fsls.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from fs.opener import opener -from fs.path import pathsplit, abspath, isdotfile +from fs.path import pathsplit, abspath, isdotfile, iswildcard from fs.commands.runner import Command from collections import defaultdict import sys @@ -43,10 +43,10 @@ List contents of [PATH]""" path = path or '.' wildcard = None - if self.is_wildcard(path): + if iswildcard(path): path, wildcard = pathsplit(path) - if fs.isfile(path): + if path != '.' and fs.isfile(path): if not options.dirsonly: file_paths.append(path) else: diff --git a/fs/commands/fstree.py b/fs/commands/fstree.py index 36884f5..18c3335 100644 --- a/fs/commands/fstree.py +++ b/fs/commands/fstree.py @@ -14,7 +14,9 @@ Recursively display the contents of PATH in an ascii tree""" def get_optparse(self): optparse = super(FSTree, self).get_optparse() optparse.add_option('-d', '--depth', dest='depth', type="int", default=5, - help="Maximum depth to display", metavar="DEPTH") + help="Maximum depth to display", metavar="DEPTH") + optparse.add_option('-a', '--all', dest='all', action='store_true', default=False, + help="do not hide dot files") return optparse def do_run(self, options, args): @@ -31,7 +33,8 @@ Recursively display the contents of PATH in an ascii tree""" print_fs(fs, path or '', file_out=self.output_file, max_levels=options.depth, - terminal_colors=self.is_terminal()) + terminal_colors=self.is_terminal(), + hide_dotfiles=not options.all) def run(): return FSTree().run() diff --git a/fs/commands/runner.py b/fs/commands/runner.py index 9a0bb9f..cf90047 100644 --- a/fs/commands/runner.py +++ b/fs/commands/runner.py @@ -2,7 +2,7 @@ import sys from optparse import OptionParser from fs.opener import opener, OpenerError from fs.errors import FSError -from fs.path import splitext, pathsplit, isdotfile +from fs.path import splitext, pathsplit, isdotfile, iswildcard import platform from collections import defaultdict @@ -55,11 +55,6 @@ class Command(object): self.terminal_width = w self.name = self.__class__.__name__.lower() - def is_wildcard(self, path): - if path is None: - return False - return '*' in path or '?' in path - def is_terminal(self): try: return self.output_file.isatty() @@ -111,7 +106,7 @@ class Command(object): if path is None: return [], [] pathname, resourcename = pathsplit(path) - if self.is_wildcard(resourcename): + if iswildcard(resourcename): dir_paths = fs.listdir(pathname, wildcard=resourcename, absolute=True, @@ -137,7 +132,7 @@ class Command(object): resources = [] for fs, path in fs_paths: - if self.is_wildcard(path): + if path and iswildcard(path): if not files_only: dir_paths = fs.listdir(wildcard=path, dirs_only=True) for path in dir_paths: @@ -227,8 +222,8 @@ class Command(object): if self.is_terminal(): self.output("\n") return 0 - except ValueError: - pass + #except ValueError: + # pass except SystemExit: return 0 except IOError: diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 53b5aaf..eeb2d14 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -751,8 +751,7 @@ class FTPFS(FS): def __init__(self, host='', user='', passwd='', acct='', timeout=_GLOBAL_DEFAULT_TIMEOUT, port=21, - dircache=True, - max_buffer_size=128*1024*1024): + dircache=True): """ Connect to a FTP server. :param host: Host to connect to @@ -765,8 +764,7 @@ class FTPFS(FS): which will speed up operations such as getinfo, isdi, isfile, but changes to the ftp file structure will not be visible until `~fs.ftpfs.FTPFS.clear_dircache` is called - :param dircache: If True directory information will be cached for fast access - :param max_buffer_size: Number of bytes to hold before blocking write operations + :param dircache: If True directory information will be cached for fast access """ @@ -780,9 +778,7 @@ class FTPFS(FS): self.timeout = timeout self.use_dircache = dircache - self.get_dircache() - - self.max_buffer_size = max_buffer_size + self.get_dircache() self._cache_hint = False self._locals._ftp = None diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 45d4b51..1b6a938 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -156,7 +156,7 @@ class DirEntry(object): self.locks += 1
def unlock(self):
- self.locks -=1
+ self.locks -= 1
assert self.locks >=0, "Lock / Unlock mismatch!"
def desc_contents(self):
@@ -494,7 +494,7 @@ class MemoryFS(FS): if dir_entry is None:
raise ResourceNotFoundError(path)
if dir_entry.isfile():
- raise ResourceInvalidError(path,msg="that's a file, not a directory: %(path)s")
+ raise ResourceInvalidError(path, msg="not a directory: %(path)s")
paths = dir_entry.contents.keys()
for (i,p) in enumerate(paths):
if not isinstance(p,unicode):
@@ -522,7 +522,7 @@ class MemoryFS(FS): return info
@synchronize
- def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
+ def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=1024*64):
src_dir_entry = self._get_dir_entry(src)
if src_dir_entry is None:
raise ResourceNotFoundError(src)
@@ -533,7 +533,7 @@ class MemoryFS(FS): dst_dir_entry.xattrs.update(src_xattrs)
@synchronize
- def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
+ def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=1024*64):
src_dir_entry = self._get_dir_entry(src)
if src_dir_entry is None:
raise ResourceNotFoundError(src)
@@ -544,7 +544,7 @@ class MemoryFS(FS): dst_dir_entry.xattrs.update(src_xattrs)
@synchronize
- def copy(self, src, dst, overwrite=False, chunk_size=16384):
+ def copy(self, src, dst, overwrite=False, chunk_size=1024*64):
src_dir_entry = self._get_dir_entry(src)
if src_dir_entry is None:
raise ResourceNotFoundError(src)
@@ -555,7 +555,7 @@ class MemoryFS(FS): dst_dir_entry.xattrs.update(src_xattrs)
@synchronize
- def move(self, src, dst, overwrite=False, chunk_size=16384):
+ def move(self, src, dst, overwrite=False, chunk_size=1024*64):
src_dir_entry = self._get_dir_entry(src)
if src_dir_entry is None:
raise ResourceNotFoundError(src)
@@ -566,6 +566,27 @@ class MemoryFS(FS): dst_dir_entry.xattrs.update(src_xattrs)
@synchronize
+ def getcontents(self, path):
+ dir_entry = self._get_dir_entry(path)
+ if dir_entry is None:
+ raise ResourceNotFoundError(path)
+ if not dir_entry.isfile():
+ raise ResourceInvalidError(path, msg="not a directory: %(path)s")
+ return dir_entry.data or ''
+
+ @synchronize
+ def setcontents(self, path, data, chunk_size=1024*64):
+ if not isinstance(data, str):
+ return super(MemoryFS, self).setcontents(path, data, chunk_size)
+ if not self.exists(path):
+ self.open(path, 'w').close()
+
+ dir_entry = self._get_dir_entry(path)
+ if not dir_entry.isfile():
+ raise ResourceInvalidError('Not a directory %(path)s', path)
+ dir_entry.data = data
+
+ @synchronize
def setxattr(self, path, key, value):
dir_entry = self._dir_entry(path)
key = unicode(key)
diff --git a/fs/opener.py b/fs/opener.py index 4a8ff59..a6f7b66 100644 --- a/fs/opener.py +++ b/fs/opener.py @@ -1,8 +1,10 @@ import sys from fs.osfs import OSFS -from fs.path import pathsplit +from fs.path import pathsplit, basename, join, iswildcard +import os import os.path import re +from urlparse import urlparse class OpenerError(Exception): pass @@ -31,11 +33,15 @@ def _expand_syspath(path): return path + + class OpenerRegistry(object): + re_fs_url = re.compile(r''' ^ -(?:\[(.*?)\])* +(.*?) +:\/\/ (?: \((.*?)\) @@ -46,6 +52,8 @@ class OpenerRegistry(object): \+(.*?)$ )*$ ''', re.VERBOSE) + + def __init__(self, openers=[]): self.registry = {} @@ -56,13 +64,12 @@ class OpenerRegistry(object): @classmethod def split_segments(self, fs_url): - match = self.re_fs_url.match(fs_url) - assert match is not None, "broken re?" - return match.groups() + match = self.re_fs_url.match(fs_url) + return match def get_opener(self, name): if name not in self.registry: - raise NoOpenerError("No opener for [%s]" % name) + raise NoOpenerError("No opener for %s" % name) index = self.registry[name] return self.openers[index] @@ -73,34 +80,56 @@ class OpenerRegistry(object): self.registry[name] = index def parse(self, fs_url, default_fs_name=None, writeable=False, create=False): - - fs_name, paren_url, fs_url, path = self.split_segments(fs_url) + + orig_url = fs_url + match = self.split_segments(fs_url) - fs_url = fs_url or paren_url - if fs_name is None and path is None: - fs_url = os.path.expanduser(os.path.expandvars(fs_url)) - fs_url = os.path.normpath(os.path.abspath(fs_url)) - fs_url, path = pathsplit(fs_url) - if not fs_url: - fs_url = '/' + if match: + fs_name, paren_url, fs_url, path = match.groups() + fs_url = fs_url or paren_url or '' + if ':' in fs_name: + fs_name, sub_protocol = fs_name.split(':', 1) + fs_url = '%s://%s' % (sub_protocol, fs_url) + + fs_name = fs_name or self.default_opener + + else: + fs_name = default_fs_name or self.default_opener + fs_url = _expand_syspath(fs_url) + path = '' + + + fs_name, fs_name_params = self.parse_name(fs_name) + opener = self.get_opener(fs_name) - fs_name = fs_name or self.default_opener + if fs_url is None: + raise OpenerError("Unable to parse '%s'" % orig_url) - if fs_name is None: - fs_name = fs_default_name + wildcard = None + if iswildcard(fs_url): + fs_url, wildcard = pathsplit(fs_url) - fs_name, fs_name_params = self.parse_name(fs_name) - opener = self.get_opener(fs_name) - - fs = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create) + fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create) + + if wildcard: + fs_path = join(fs_path or '', wildcard) + else: + path = join(fs_path or '', path) if path: pathname, resourcename = pathsplit(path) if pathname: fs = fs.opendir(pathname) path = resourcename + if not iswildcard(path): + if fs.isdir(path): + fs = fs.opendir(path) + fs_path = '' + else: + fs_path = path + - return fs, path + return fs, fs_path def parse_credentials(self, url): @@ -145,11 +174,21 @@ class OSFSOpener(Opener): names = ['osfs', 'file'] @classmethod - def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): - username, password, fs_path = registry.parse_credentials(fs_path) - from fs.osfs import OSFS - osfs = OSFS(fs_path, create=create) - return osfs + def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): + from fs.osfs import OSFS + username, password, fs_path = registry.parse_credentials(fs_path) + + + path = _expand_syspath(fs_path) + if create: + sys.makedirs(fs_path) + if os.path.isdir(path): + osfs = OSFS(path) + filepath = None + else: + path, filepath = pathsplit(path) + osfs = OSFS(path, create=create) + return osfs, filepath class ZipOpener(Opener): @@ -157,22 +196,21 @@ class ZipOpener(Opener): @classmethod def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): - - create_zip = fs_name_params == 'new' - append_zip = fs_name_params == 'append' - - zip_file = None - if fs_path.startswith('['): - container_fs, container_path = registry.parse(fs_path) - if not container_path: - raise OpenerError("Not a file") - container_mode = 'r+b' - if create_zip: - container_mode = 'w+b' - elif writeable: - container_mode = 'w+b' + + append_zip = fs_name_params == 'add' - zip_file = container_fs.open(container_path, mode=container_mode) + zip_fs, zip_path = registry.parse(fs_path) + if zip_path is None: + raise OpenerError('File required for zip opener') + if create: + open_mode = 'wb' + if append_zip: + open_mode = 'r+b' + else: + open_mode = 'rb' + + zip_file = zip_fs.open(zip_path, mode=open_mode) + username, password, fs_path = registry.parse_credentials(fs_path) @@ -182,21 +220,18 @@ class ZipOpener(Opener): if append_zip: mode = 'a' - elif create_zip or create: + elif create: mode = 'w' else: if writeable: mode = 'w' else: mode = 'a' - - if fs_name == 'zip64': - allow_zip_64 = True - else: - allow_zip_64 = False + + allow_zip_64 = fs_name == 'zip64' zipfs = ZipFS(zip_file, mode=mode, allow_zip_64=allow_zip_64) - return zipfs + return zipfs, None class RPCOpener(Opener): names = ['rpc'] @@ -206,9 +241,16 @@ class RPCOpener(Opener): from fs.rpcfs import RPCFS username, password, fs_path = registry.parse_credentials(fs_path) if not fs_path.startswith('http://'): - fs_path = 'http://' + fs_path - rpcfs = RPCFS(fs_path) - return rpcfs + fs_path = 'http://' + fs_path + + scheme, netloc, path, params, query, fragment = urlparse(fs_path) + + rpcfs = RPCFS('%s://%s' % (scheme, netloc)) + + if create and path: + rpcfs.makedir(path, recursive=True, allow_recreate=True) + + return rpcfs, path or None class FTPOpener(Opener): names = ['ftp'] @@ -217,22 +259,28 @@ class FTPOpener(Opener): def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): from fs.ftpfs import FTPFS username, password, fs_path = registry.parse_credentials(fs_path) - - if '/' in fs_path: - url, root_path = fs_path.split('/', 1) - else: - url = fs_path - root_path = '' - + + scheme, netloc, path, params, query, fragment = urlparse(fs_path) + if not scheme: + fs_path = 'ftp://' + fs_path + scheme, netloc, path, params, query, fragment = urlparse(fs_path) + + dirpath, resourcepath = pathsplit(path) + url = netloc + ftpfs = FTPFS(url, user=username or '', passwd=password or '') ftpfs.cache_hint(True) - if root_path not in ('', '/'): - if not ftpfs.isdir(root_path): - raise OpenerError("'%s' is not a directory on the server" % root_path) - return ftpfs.opendir(root_path) + if create and path: + ftpfs.makedir(path, recursive=True, allow_recreate=True) - return ftpfs + if dirpath: + ftpfs = ftpfs.opendir(dirpath) + + if not resourcepath: + return ftpfs, None + else: + return ftpfs, resourcepath class SFTPOpener(Opener): @@ -256,6 +304,8 @@ class SFTPOpener(Opener): addr = fs_path fs_path = '/' + fs_path, resourcename = pathsplit(fs_path) + host = addr port = None if ':' in host: @@ -265,10 +315,25 @@ class SFTPOpener(Opener): except ValueError: pass else: - host = (addr, port) + host = (addr, port) + + #if not username or not password: + # raise OpenerError('SFTP requires authentication') + + if create: + sftpfs = SFTPFS(host, root_path='/', **credentials) + if not sftpfs._transport.is_authenticated(): + sftpfs.close() + raise OpenerError('SFTP requires authentication') + sftpfs = sfspfs.makeopendir(fs_path) + return sftpfs, None sftpfs = SFTPFS(host, root_path=fs_path, **credentials) - return sftpfs + if not sftpfs._transport.is_authenticated(): + sftpfs.close() + raise OpenerError('SFTP requires authentication') + + return sftpfs, resourcename class MemOpener(Opener): @@ -277,7 +342,10 @@ class MemOpener(Opener): @classmethod def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): from fs.memoryfs import MemoryFS - return MemoryFS() + memfs = MemoryFS() + if create: + memfs = memfs.makeopendir(fs_path) + return memfs, None class DebugOpener(Opener): names = ['debug'] @@ -287,13 +355,13 @@ class DebugOpener(Opener): from fs.wrapfs.debugfs import DebugFS if fs_path: fs, path = registry.parse(fs_path, writeable=writeable, create=create) - return DebugFS(fs, verbose=False) + return DebugFS(fs, verbose=False), None if fs_name_params == 'ram': from fs.memoryfs import MemoryFS - return DebugFS(MemoryFS(), identifier=fs_name_params, verbose=False) + return DebugFS(MemoryFS(), identifier=fs_name_params, verbose=False), None else: from fs.tempfs import TempFS - return DebugFS(TempFS(), identifier=fs_name_params, verbose=False) + return DebugFS(TempFS(), identifier=fs_name_params, verbose=False), None class TempOpener(Opener): names = ['temp'] @@ -301,7 +369,7 @@ class TempOpener(Opener): @classmethod def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): from fs.tempfs import TempFS - return TempFS(identifier=fs_name_params, temp_dir=fs_path) + return TempFS(identifier=fs_name_params, temp_dir=fs_path), None opener = OpenerRegistry([OSFSOpener, @@ -317,7 +385,7 @@ opener = OpenerRegistry([OSFSOpener, def main(): - fs, path = opener.parse('galleries.zip') + fs, path = opener.parse('sftp://willmcgugan.com') print fs, path if __name__ == "__main__": @@ -515,3 +515,16 @@ class PathMap(object): def names(self,root="/"): return list(self.iternames(root)) +_wild_chars = frozenset('*?[]!{}') +def iswildcard(path): + """Check if a path ends with a wildcard + + >>> is_wildcard('foo/bar/baz.*') + True + >>> is_wildcard('foo/bar') + False + + """ + assert path is not None + base_chars = frozenset(basename(path)) + return not base_chars.isdisjoint(_wild_chars) diff --git a/fs/sftpfs.py b/fs/sftpfs.py index 171d543..76308c2 100644 --- a/fs/sftpfs.py +++ b/fs/sftpfs.py @@ -9,12 +9,15 @@ Filesystem accessing an SFTP server (via paramiko) import datetime import stat as statinfo import threading - +import os import paramiko +from getpass import getuser +from binascii import hexlify from fs.base import * from fs.path import * from fs.errors import * +from fs.utils import isdir, isfile # SFTPClient appears to not be thread-safe, so we use an instance per thread if hasattr(threading, "local"): @@ -58,6 +61,7 @@ class SFTPFS(FS): 'atomic.setcontents' : False } + def __init__(self, connection, root_path="/", encoding=None, **credentials): """SFTPFS constructor. @@ -88,18 +92,81 @@ class SFTPFS(FS): self._tlocal = thread_local() self._transport = None self._client = None + + + hostname = None + if isinstance(connection, basestring): + hostname = connection + else: + try: + hostname, port = connection + except ValueError: + pass + + + hostkeytype = None + hostkey = None + + if hostname is not None: + try: + host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts')) + except IOError: + try: + # try ~/ssh/ too, because windows can't have a folder named ~/.ssh/ + host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/ssh/known_hosts')) + except IOError: + + host_keys = {} + + if host_keys.has_key(hostname): + hostkeytype = host_keys[hostname].keys()[0] + hostkey = host_keys[hostname][hostkeytype] + credentials['hostkey'] = hostkey + + if not credentials.get('username'): + credentials['username'] = getuser() + + super(SFTPFS, self).__init__() + if isinstance(connection,paramiko.Channel): self._transport = None self._client = paramiko.SFTPClient(connection) - else: - if not isinstance(connection,paramiko.Transport): + else: + if not isinstance(connection, paramiko.Transport): connection = paramiko.Transport(connection) self._owns_transport = True - if not connection.is_authenticated(): - connection.connect(**credentials) + try: + if not connection.is_authenticated(): + connection.connect(**credentials) + if not connection.is_authenticated(): + self._agent_auth(connection, credentials.get('username')) + if not connection.is_authenticated(): + connection.close() + raise RemoteConnectionError('No auth') + except paramiko.AuthenticationException: + raise RemoteConnectionError('Auth rejected') self._transport = connection self.root_path = abspath(normpath(root_path)) - super(SFTPFS, self).__init__() + + @classmethod + def _agent_auth(cls, transport, username): + """ + Attempt to authenticate to the given transport using any of the private + keys available from an SSH agent. + """ + + agent = paramiko.Agent() + agent_keys = agent.get_keys() + if len(agent_keys) == 0: + return False + + for key in agent_keys: + try: + transport.auth_publickey(username, key) + return key + except paramiko.SSHException: + pass + return None def __del__(self): self.close() @@ -184,6 +251,8 @@ class SFTPFS(FS): @convert_os_errors def isdir(self,path): + if path == '/': + return True npath = self._normpath(path) try: stat = self.client.stat(npath) @@ -209,6 +278,10 @@ class SFTPFS(FS): npath = self._normpath(path) try: paths = self.client.listdir(npath) + if dirs_only or files_only: + path_attrs = self.client.listdir_attr(npath) + else: + path_attrs = None except IOError, e: if getattr(e,"errno",None) == 2: if self.isfile(path): @@ -217,10 +290,72 @@ class SFTPFS(FS): elif self.isfile(path): raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s") raise + + if path_attrs is not None: + if dirs_only: + filter_paths = [] + for path, attr in zip(paths, path_attrs): + if isdir(self, path, attr.__dict__): + filter_paths.append(path) + paths = filter_paths + elif files_only: + filter_paths = [] + for path, attr in zip(paths, path_attrs): + if isfile(self, path, attr.__dict__): + filter_paths.append(path) + paths = filter_paths + for (i,p) in enumerate(paths): if not isinstance(p,unicode): paths[i] = p.decode(self.encoding) - return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only) + return self._listdir_helper(path, paths, wildcard, full, absolute, False, False) + + + @convert_os_errors + def listdirinfo(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False): + npath = self._normpath(path) + try: + paths = self.client.listdir(npath) + attrs = self.client.listdir_attr(npath) + attrs_map = dict(zip(paths, attrs)) + except IOError, e: + if getattr(e,"errno",None) == 2: + if self.isfile(path): + raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s") + raise ResourceNotFoundError(path) + elif self.isfile(path): + raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s") + raise + + + if dirs_only: + filter_paths = [] + for path, attr in zip(paths, attrs): + if isdir(self, path, attr.__dict__): + filter_paths.append(path) + paths = filter_paths + elif files_only: + filter_paths = [] + for path, attr in zip(paths, attrs): + if isfile(self, path, attr.__dict__): + filter_paths.append(path) + paths = filter_paths + + for (i, p) in enumerate(paths): + if not isinstance(p, unicode): + paths[i] = p.decode(self.encoding) + + def getinfo(p): + resourcename = basename(p) + info = attrs_map.get(resourcename) + if info is None: + return self.getinfo(pathjoin(path, p)) + return self._extract_info(info.__dict__) + + return [(p, getinfo(p)) for p in + self._listdir_helper(path, paths, wildcard, full, absolute, False, False)] + + @convert_os_errors def makedir(self,path,recursive=False,allow_recreate=False): @@ -335,6 +470,23 @@ class SFTPFS(FS): raise ParentDirectoryMissingError(dst,msg="Destination directory does not exist: %(path)s") raise + _info_vars = frozenset('st_size st_uid st_gid st_mode st_atime st_mtime'.split()) + @classmethod + def _extract_info(cls, stats): + fromtimestamp = datetime.datetime.fromtimestamp + info = dict((k, v) for k, v in stats.iteritems() if k in cls._info_vars) + info['size'] = info['st_size'] + ct = info.get('st_ctime') + if ct is not None: + info['created_time'] = fromtimestamp(ct) + at = info.get('st_atime') + if at is not None: + info['accessed_time'] = fromtimestamp(at) + mt = info.get('st_mtime') + if mt is not None: + info['modified_time'] = fromtimestamp(mt) + return info + @convert_os_errors def getinfo(self, path): npath = self._normpath(path) diff --git a/fs/utils.py b/fs/utils.py index 887cfa3..4035d6c 100644 --- a/fs/utils.py +++ b/fs/utils.py @@ -56,10 +56,13 @@ def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1 src = None try: # Chunk copy - src = src_fs.open(src_path, 'rb') + if src_fs.getsize(src_path) < chunk_size: + src = src_fs.getcontents(src_path) + else: + src = src_fs.open(src_path, 'rb') dst_fs.setcontents(dst_path, src, chunk_size=chunk_size) finally: - if src is not None: + if src is not None and hasattr(src, 'close'): src.close() @@ -89,14 +92,18 @@ def movefile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1 FS._shutil_movefile(src_syspath, dst_syspath) return - src = None - try: - src = src_fs.open(src_path, 'rb') - dst_fs.setcontents(dst_path, src, chunk_size=chunk_size) - src_fs.remove(src_path) + src = None + try: + # Chunk copy + if src_fs.getsize(src_path) < chunk_size: + src = src_fs.getcontents(src_path) + else: + src = src_fs.open(src_path, 'rb') + dst_fs.setcontents(dst_path, src, chunk_size=chunk_size) + src_fs.remove(src_path) finally: - if src is not None: + if src is not None and hasattr(src, 'close'): src.close() def movedir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024): @@ -324,7 +331,7 @@ def find_duplicates(fs, paths = list(set(paths).difference(dups)) -def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None): +def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hide_dotfiles=False): """Prints a filesystem listing to stdout (including sub dirs). Useful as a debugging aid. Be careful about printing a OSFS, or any other large filesystem. Without max_levels set, this function will traverse the entire directory tree. @@ -343,13 +350,14 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None): :param file_out: File object to write output to (defaults to sys.stdout) :param terminal_colors: If True, terminal color codes will be written, set to False for non-console output. The default (None) will select an appropriate setting for the platform. + :param hide_dotfiles: if True, files or directories begining with '.' will be removed """ if file_out is None: file_out = sys.stdout - file_encoding = getattr(file_out, 'encoding', 'utf-8') + file_encoding = getattr(file_out, 'encoding', 'utf-8') or 'utf-8' if terminal_colors is None: if sys.platform.startswith('win'): @@ -388,12 +396,16 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None): def print_dir(fs, path, levels=[]): try: - dir_listing = [(fs.isdir(pathjoin(path,p)), p) for p in fs.listdir(path)] + dir_listing = ( [(True, p) for p in fs.listdir(path, dirs_only=True)] + + [(False, p) for p in fs.listdir(path, files_only=True)] ) except Exception, e: prefix = ''.join([('| ', ' ')[last] for last in levels]) + ' ' write(wrap_prefix(prefix[:-1] + ' ') + wrap_error("unabled to retrieve directory list (%s) ..." % str(e))) return 0 + if hide_dotfiles: + dir_listing = [(isdir, p) for isdir, p in dir_listing if not p.startswith('.')] + dir_listing.sort(key = lambda (isdir, p):(not isdir, p.lower())) for i, (is_dir, item) in enumerate(dir_listing): |