diff options
Diffstat (limited to 'fs/contrib/tahoelafs')
| -rw-r--r-- | fs/contrib/tahoelafs/__init__.py | 830 | ||||
| -rw-r--r-- | fs/contrib/tahoelafs/connection.py | 24 | ||||
| -rw-r--r-- | fs/contrib/tahoelafs/test_tahoelafs.py | 104 | ||||
| -rw-r--r-- | fs/contrib/tahoelafs/util.py | 280 |
4 files changed, 619 insertions, 619 deletions
diff --git a/fs/contrib/tahoelafs/__init__.py b/fs/contrib/tahoelafs/__init__.py index c01dd5e..91858bd 100644 --- a/fs/contrib/tahoelafs/__init__.py +++ b/fs/contrib/tahoelafs/__init__.py @@ -1,415 +1,415 @@ -'''
-fs.contrib.tahoelafs
-====================
-
-This modules provides a PyFilesystem interface to the Tahoe Least Authority
-File System. Tahoe-LAFS is a distributed, encrypted, fault-tolerant storage
-system:
-
- http://tahoe-lafs.org/
-
-You will need access to a Tahoe-LAFS "web api" service.
-
-Example (it will use publicly available (but slow) Tahoe-LAFS cloud)::
-
- from fs.contrib.tahoelafs import TahoeLAFS, Connection
- dircap = TahoeLAFS.createdircap(webapi='http://insecure.tahoe-lafs.org')
- print "Your dircap (unique key to your storage directory) is", dircap
- print "Keep it safe!"
- fs = TahoeLAFS(dircap, autorun=False, webapi='http://insecure.tahoe-lafs.org')
- f = fs.open("foo.txt", "a")
- f.write('bar!')
- f.close()
- print "Now visit %s and enjoy :-)" % fs.getpathurl('foo.txt')
-
-When any problem occurred, you can turn on internal debugging messages::
-
- import logging
- l = logging.getLogger()
- l.setLevel(logging.DEBUG)
- l.addHandler(logging.StreamHandler(sys.stdout))
-
- ... your Python code using TahoeLAFS ...
-
-TODO:
-
- * unicode support
- * try network errors / bad happiness
- * exceptions
- * tests
- * sanitize all path types (., /)
- * support for extra large file uploads (poster module)
- * Possibility to block write until upload done (Tahoe mailing list)
- * Report something sane when Tahoe crashed/unavailable
- * solve failed unit tests (makedir_winner, ...)
- * file times
- * docs & author
- * python3 support
- * remove creating blank files (depends on FileUploadManager)
-
-TODO (Not TahoeLAFS specific tasks):
- * RemoteFileBuffer on the fly buffering support
- * RemoteFileBuffer unit tests
- * RemoteFileBuffer submit to trunk
- * Implement FileUploadManager + faking isfile/exists of just processing file
- * pyfilesystem docs is outdated (rename, movedir, ...)
-
-'''
-
-
-import stat as statinfo
-
-import logging
-from logging import DEBUG, INFO, ERROR, CRITICAL
-
-import fs
-import fs.errors as errors
-from fs.path import abspath, relpath, normpath, dirname, pathjoin
-from fs.base import FS, NullFile
-from fs import _thread_synchronize_default, SEEK_END
-from fs.remote import CacheFSMixin, RemoteFileBuffer
-from fs.base import fnmatch, NoDefaultMeta
-
-from util import TahoeUtil
-from connection import Connection
-
-from six import b
-
-logger = fs.getLogger('fs.tahoelafs')
-
-def _fix_path(func):
- """Method decorator for automatically normalising paths."""
- def wrapper(self, *args, **kwds):
- if len(args):
- args = list(args)
- args[0] = _fixpath(args[0])
- return func(self, *args, **kwds)
- return wrapper
-
-
-def _fixpath(path):
- """Normalize the given path."""
- return abspath(normpath(path))
-
-
-
-class _TahoeLAFS(FS):
- """FS providing raw access to a Tahoe-LAFS Filesystem.
-
- This class implements all the details of interacting with a Tahoe-backed
- filesystem, but you probably don't want to use it in practice. Use the
- TahoeLAFS class instead, which has some internal caching to improve
- performance.
- """
-
- _meta = { 'virtual' : False,
- 'read_only' : False,
- 'unicode_paths' : True,
- 'case_insensitive_paths' : False,
- 'network' : True
- }
-
-
- def __init__(self, dircap, largefilesize=10*1024*1024, webapi='http://127.0.0.1:3456'):
- '''Creates instance of TahoeLAFS.
-
- :param dircap: special hash allowing user to work with TahoeLAFS directory.
- :param largefilesize: - Create placeholder file for files larger than this treshold.
- Uploading and processing of large files can last extremely long (many hours),
- so placing this placeholder can help you to remember that upload is processing.
- Setting this to None will skip creating placeholder files for any uploads.
- '''
- self.dircap = dircap if not dircap.endswith('/') else dircap[:-1]
- self.largefilesize = largefilesize
- self.connection = Connection(webapi)
- self.tahoeutil = TahoeUtil(webapi)
- super(_TahoeLAFS, self).__init__(thread_synchronize=_thread_synchronize_default)
-
- def __str__(self):
- return "<TahoeLAFS: %s>" % self.dircap
-
- @classmethod
- def createdircap(cls, webapi='http://127.0.0.1:3456'):
- return TahoeUtil(webapi).createdircap()
-
- def getmeta(self,meta_name,default=NoDefaultMeta):
- if meta_name == "read_only":
- return self.dircap.startswith('URI:DIR2-RO')
- return super(_TahoeLAFS,self).getmeta(meta_name,default)
-
- @_fix_path
- def open(self, path, mode='r', **kwargs):
- self._log(INFO, 'Opening file %s in mode %s' % (path, mode))
- newfile = False
- if not self.exists(path):
- if 'w' in mode or 'a' in mode:
- newfile = True
- else:
- self._log(DEBUG, "File %s not found while opening for reads" % path)
- raise errors.ResourceNotFoundError(path)
- elif self.isdir(path):
- self._log(DEBUG, "Path %s is directory, not a file" % path)
- raise errors.ResourceInvalidError(path)
- elif 'w' in mode:
- newfile = True
-
- if newfile:
- self._log(DEBUG, 'Creating empty file %s' % path)
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- self.setcontents(path, b(''))
- handler = NullFile()
- else:
- self._log(DEBUG, 'Opening existing file %s for reading' % path)
- handler = self.getrange(path,0)
-
- return RemoteFileBuffer(self, path, mode, handler,
- write_on_flush=False)
-
- @_fix_path
- def desc(self, path):
- try:
- return self.getinfo(path)
- except:
- return ''
-
- @_fix_path
- def exists(self, path):
- try:
- self.getinfo(path)
- self._log(DEBUG, "Path %s exists" % path)
- return True
- except errors.ResourceNotFoundError:
- self._log(DEBUG, "Path %s does not exists" % path)
- return False
- except errors.ResourceInvalidError:
- self._log(DEBUG, "Path %s does not exists, probably misspelled URI" % path)
- return False
-
- @_fix_path
- def getsize(self, path):
- try:
- size = self.getinfo(path)['size']
- self._log(DEBUG, "Size of %s is %d" % (path, size))
- return size
- except errors.ResourceNotFoundError:
- return 0
-
- @_fix_path
- def isfile(self, path):
- try:
- isfile = (self.getinfo(path)['type'] == 'filenode')
- except errors.ResourceNotFoundError:
- #isfile = not path.endswith('/')
- isfile = False
- self._log(DEBUG, "Path %s is file: %d" % (path, isfile))
- return isfile
-
- @_fix_path
- def isdir(self, path):
- try:
- isdir = (self.getinfo(path)['type'] == 'dirnode')
- except errors.ResourceNotFoundError:
- isdir = False
- self._log(DEBUG, "Path %s is directory: %d" % (path, isdir))
- return isdir
-
-
- def listdir(self, *args, **kwargs):
- return [ item[0] for item in self.listdirinfo(*args, **kwargs) ]
-
- def listdirinfo(self, *args, **kwds):
- return list(self.ilistdirinfo(*args,**kwds))
-
- def ilistdir(self, *args, **kwds):
- for item in self.ilistdirinfo(*args,**kwds):
- yield item[0]
-
- @_fix_path
- def ilistdirinfo(self, path="/", wildcard=None, full=False, absolute=False,
- dirs_only=False, files_only=False):
- self._log(DEBUG, "Listing directory (listdirinfo) %s" % path)
-
- if dirs_only and files_only:
- raise ValueError("dirs_only and files_only can not both be True")
-
- for item in self.tahoeutil.list(self.dircap, path):
- if dirs_only and item['type'] == 'filenode':
- continue
- elif files_only and item['type'] == 'dirnode':
- continue
-
- if wildcard is not None:
- if isinstance(wildcard,basestring):
- if not fnmatch.fnmatch(item['name'], wildcard):
- continue
- else:
- if not wildcard(item['name']):
- continue
-
- if full:
- item_path = relpath(pathjoin(path, item['name']))
- elif absolute:
- item_path = abspath(pathjoin(path, item['name']))
- else:
- item_path = item['name']
-
- yield (item_path, item)
-
- @_fix_path
- def remove(self, path):
- self._log(INFO, 'Removing file %s' % path)
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
-
- if not self.isfile(path):
- if not self.isdir(path):
- raise errors.ResourceNotFoundError(path)
- raise errors.ResourceInvalidError(path)
-
- try:
- self.tahoeutil.unlink(self.dircap, path)
- except Exception, e:
- raise errors.ResourceInvalidError(path)
-
- @_fix_path
- def removedir(self, path, recursive=False, force=False):
- self._log(INFO, "Removing directory %s" % path)
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- if not self.isdir(path):
- if not self.isfile(path):
- raise errors.ResourceNotFoundError(path)
- raise errors.ResourceInvalidError(path)
- if not force and self.listdir(path):
- raise errors.DirectoryNotEmptyError(path)
-
- self.tahoeutil.unlink(self.dircap, path)
-
- if recursive and path != '/':
- try:
- self.removedir(dirname(path), recursive=True)
- except errors.DirectoryNotEmptyError:
- pass
-
- @_fix_path
- def makedir(self, path, recursive=False, allow_recreate=False):
- self._log(INFO, "Creating directory %s" % path)
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- if self.exists(path):
- if not self.isdir(path):
- raise errors.ResourceInvalidError(path)
- if not allow_recreate:
- raise errors.DestinationExistsError(path)
- if not recursive and not self.exists(dirname(path)):
- raise errors.ParentDirectoryMissingError(path)
- self.tahoeutil.mkdir(self.dircap, path)
-
- def movedir(self, src, dst, overwrite=False):
- self.move(src, dst, overwrite=overwrite)
-
- def move(self, src, dst, overwrite=False):
- self._log(INFO, "Moving file from %s to %s" % (src, dst))
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- src = _fixpath(src)
- dst = _fixpath(dst)
- if not self.exists(dirname(dst)):
- raise errors.ParentDirectoryMissingError(dst)
- if not overwrite and self.exists(dst):
- raise errors.DestinationExistsError(dst)
- self.tahoeutil.move(self.dircap, src, dst)
-
- def rename(self, src, dst):
- self.move(src, dst)
-
- def copy(self, src, dst, overwrite=False, chunk_size=16384):
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- # FIXME: this is out of date; how to do native tahoe copy?
- # FIXME: Workaround because isfile() not exists on _TahoeLAFS
- FS.copy(self, src, dst, overwrite, chunk_size)
-
- def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- # FIXME: this is out of date; how to do native tahoe copy?
- # FIXME: Workaround because isfile() not exists on _TahoeLAFS
- FS.copydir(self, src, dst, overwrite, ignore_errors, chunk_size)
-
-
- def _log(self, level, message):
- if not logger.isEnabledFor(level): return
- logger.log(level, u'(%d) %s' % (id(self),
- unicode(message).encode('ASCII', 'replace')))
-
- @_fix_path
- def getpathurl(self, path, allow_none=False, webapi=None):
- '''
- Retrieve URL where the file/directory is stored
- '''
- if webapi == None:
- webapi = self.connection.webapi
- self._log(DEBUG, "Retrieving URL for %s over %s" % (path, webapi))
- path = self.tahoeutil.fixwinpath(path, False)
- return u"%s/uri/%s%s" % (webapi, self.dircap, path)
-
- @_fix_path
- def getrange(self, path, offset, length=None):
- return self.connection.get(u'/uri/%s%s' % (self.dircap, path),
- offset=offset, length=length)
-
- @_fix_path
- def setcontents(self, path, file, chunk_size=64*1024):
- self._log(INFO, 'Uploading file %s' % path)
- size=None
-
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
-
- # Workaround for large files:
- # First create zero file placeholder, then
- # upload final content.
- if self.largefilesize != None and getattr(file, 'read', None):
- # As 'file' can be also a string, need to check,
- # if 'file' looks like duck. Sorry, file.
- file.seek(0, SEEK_END)
- size = file.tell()
- file.seek(0)
-
- if size > self.largefilesize:
- self.connection.put(u'/uri/%s%s' % (self.dircap, path),
- "PyFilesystem.TahoeLAFS: Upload started, final size %d" % size)
-
- self.connection.put(u'/uri/%s%s' % (self.dircap, path), file, size=size)
-
- @_fix_path
- def getinfo(self, path):
- self._log(INFO, 'Reading meta for %s' % path)
- info = self.tahoeutil.info(self.dircap, path)
- #import datetime
- #info['created_time'] = datetime.datetime.now()
- #info['modified_time'] = datetime.datetime.now()
- #info['accessed_time'] = datetime.datetime.now()
- if info['type'] == 'filenode':
- info["st_mode"] = 0x700 | statinfo.S_IFREG
- elif info['type'] == 'dirnode':
- info["st_mode"] = 0x700 | statinfo.S_IFDIR
- return info
-
-
-
-class TahoeLAFS(CacheFSMixin,_TahoeLAFS):
- """FS providing cached access to a Tahoe Filesystem.
-
- This class is the preferred means to access a Tahoe filesystem. It
- maintains an internal cache of recently-accessed metadata to speed
- up operations.
- """
-
- def __init__(self, *args, **kwds):
- kwds.setdefault("cache_timeout",60)
- super(TahoeLAFS,self).__init__(*args,**kwds)
-
-
+''' +fs.contrib.tahoelafs +==================== + +This modules provides a PyFilesystem interface to the Tahoe Least Authority +File System. Tahoe-LAFS is a distributed, encrypted, fault-tolerant storage +system: + + http://tahoe-lafs.org/ + +You will need access to a Tahoe-LAFS "web api" service. + +Example (it will use publicly available (but slow) Tahoe-LAFS cloud):: + + from fs.contrib.tahoelafs import TahoeLAFS, Connection + dircap = TahoeLAFS.createdircap(webapi='http://insecure.tahoe-lafs.org') + print "Your dircap (unique key to your storage directory) is", dircap + print "Keep it safe!" + fs = TahoeLAFS(dircap, autorun=False, webapi='http://insecure.tahoe-lafs.org') + f = fs.open("foo.txt", "a") + f.write('bar!') + f.close() + print "Now visit %s and enjoy :-)" % fs.getpathurl('foo.txt') + +When any problem occurred, you can turn on internal debugging messages:: + + import logging + l = logging.getLogger() + l.setLevel(logging.DEBUG) + l.addHandler(logging.StreamHandler(sys.stdout)) + + ... your Python code using TahoeLAFS ... + +TODO: + + * unicode support + * try network errors / bad happiness + * exceptions + * tests + * sanitize all path types (., /) + * support for extra large file uploads (poster module) + * Possibility to block write until upload done (Tahoe mailing list) + * Report something sane when Tahoe crashed/unavailable + * solve failed unit tests (makedir_winner, ...) + * file times + * docs & author + * python3 support + * remove creating blank files (depends on FileUploadManager) + +TODO (Not TahoeLAFS specific tasks): + * RemoteFileBuffer on the fly buffering support + * RemoteFileBuffer unit tests + * RemoteFileBuffer submit to trunk + * Implement FileUploadManager + faking isfile/exists of just processing file + * pyfilesystem docs is outdated (rename, movedir, ...) + +''' + + +import stat as statinfo + +import logging +from logging import DEBUG, INFO, ERROR, CRITICAL + +import fs +import fs.errors as errors +from fs.path import abspath, relpath, normpath, dirname, pathjoin +from fs.base import FS, NullFile +from fs import _thread_synchronize_default, SEEK_END +from fs.remote import CacheFSMixin, RemoteFileBuffer +from fs.base import fnmatch, NoDefaultMeta + +from util import TahoeUtil +from connection import Connection + +from six import b + +logger = fs.getLogger('fs.tahoelafs') + +def _fix_path(func): + """Method decorator for automatically normalising paths.""" + def wrapper(self, *args, **kwds): + if len(args): + args = list(args) + args[0] = _fixpath(args[0]) + return func(self, *args, **kwds) + return wrapper + + +def _fixpath(path): + """Normalize the given path.""" + return abspath(normpath(path)) + + + +class _TahoeLAFS(FS): + """FS providing raw access to a Tahoe-LAFS Filesystem. + + This class implements all the details of interacting with a Tahoe-backed + filesystem, but you probably don't want to use it in practice. Use the + TahoeLAFS class instead, which has some internal caching to improve + performance. + """ + + _meta = { 'virtual' : False, + 'read_only' : False, + 'unicode_paths' : True, + 'case_insensitive_paths' : False, + 'network' : True + } + + + def __init__(self, dircap, largefilesize=10*1024*1024, webapi='http://127.0.0.1:3456'): + '''Creates instance of TahoeLAFS. + + :param dircap: special hash allowing user to work with TahoeLAFS directory. + :param largefilesize: - Create placeholder file for files larger than this treshold. + Uploading and processing of large files can last extremely long (many hours), + so placing this placeholder can help you to remember that upload is processing. + Setting this to None will skip creating placeholder files for any uploads. + ''' + self.dircap = dircap if not dircap.endswith('/') else dircap[:-1] + self.largefilesize = largefilesize + self.connection = Connection(webapi) + self.tahoeutil = TahoeUtil(webapi) + super(_TahoeLAFS, self).__init__(thread_synchronize=_thread_synchronize_default) + + def __str__(self): + return "<TahoeLAFS: %s>" % self.dircap + + @classmethod + def createdircap(cls, webapi='http://127.0.0.1:3456'): + return TahoeUtil(webapi).createdircap() + + def getmeta(self,meta_name,default=NoDefaultMeta): + if meta_name == "read_only": + return self.dircap.startswith('URI:DIR2-RO') + return super(_TahoeLAFS,self).getmeta(meta_name,default) + + @_fix_path + def open(self, path, mode='r', **kwargs): + self._log(INFO, 'Opening file %s in mode %s' % (path, mode)) + newfile = False + if not self.exists(path): + if 'w' in mode or 'a' in mode: + newfile = True + else: + self._log(DEBUG, "File %s not found while opening for reads" % path) + raise errors.ResourceNotFoundError(path) + elif self.isdir(path): + self._log(DEBUG, "Path %s is directory, not a file" % path) + raise errors.ResourceInvalidError(path) + elif 'w' in mode: + newfile = True + + if newfile: + self._log(DEBUG, 'Creating empty file %s' % path) + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + self.setcontents(path, b('')) + handler = NullFile() + else: + self._log(DEBUG, 'Opening existing file %s for reading' % path) + handler = self.getrange(path,0) + + return RemoteFileBuffer(self, path, mode, handler, + write_on_flush=False) + + @_fix_path + def desc(self, path): + try: + return self.getinfo(path) + except: + return '' + + @_fix_path + def exists(self, path): + try: + self.getinfo(path) + self._log(DEBUG, "Path %s exists" % path) + return True + except errors.ResourceNotFoundError: + self._log(DEBUG, "Path %s does not exists" % path) + return False + except errors.ResourceInvalidError: + self._log(DEBUG, "Path %s does not exists, probably misspelled URI" % path) + return False + + @_fix_path + def getsize(self, path): + try: + size = self.getinfo(path)['size'] + self._log(DEBUG, "Size of %s is %d" % (path, size)) + return size + except errors.ResourceNotFoundError: + return 0 + + @_fix_path + def isfile(self, path): + try: + isfile = (self.getinfo(path)['type'] == 'filenode') + except errors.ResourceNotFoundError: + #isfile = not path.endswith('/') + isfile = False + self._log(DEBUG, "Path %s is file: %d" % (path, isfile)) + return isfile + + @_fix_path + def isdir(self, path): + try: + isdir = (self.getinfo(path)['type'] == 'dirnode') + except errors.ResourceNotFoundError: + isdir = False + self._log(DEBUG, "Path %s is directory: %d" % (path, isdir)) + return isdir + + + def listdir(self, *args, **kwargs): + return [ item[0] for item in self.listdirinfo(*args, **kwargs) ] + + def listdirinfo(self, *args, **kwds): + return list(self.ilistdirinfo(*args,**kwds)) + + def ilistdir(self, *args, **kwds): + for item in self.ilistdirinfo(*args,**kwds): + yield item[0] + + @_fix_path + def ilistdirinfo(self, path="/", wildcard=None, full=False, absolute=False, + dirs_only=False, files_only=False): + self._log(DEBUG, "Listing directory (listdirinfo) %s" % path) + + if dirs_only and files_only: + raise ValueError("dirs_only and files_only can not both be True") + + for item in self.tahoeutil.list(self.dircap, path): + if dirs_only and item['type'] == 'filenode': + continue + elif files_only and item['type'] == 'dirnode': + continue + + if wildcard is not None: + if isinstance(wildcard,basestring): + if not fnmatch.fnmatch(item['name'], wildcard): + continue + else: + if not wildcard(item['name']): + continue + + if full: + item_path = relpath(pathjoin(path, item['name'])) + elif absolute: + item_path = abspath(pathjoin(path, item['name'])) + else: + item_path = item['name'] + + yield (item_path, item) + + @_fix_path + def remove(self, path): + self._log(INFO, 'Removing file %s' % path) + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + + if not self.isfile(path): + if not self.isdir(path): + raise errors.ResourceNotFoundError(path) + raise errors.ResourceInvalidError(path) + + try: + self.tahoeutil.unlink(self.dircap, path) + except Exception, e: + raise errors.ResourceInvalidError(path) + + @_fix_path + def removedir(self, path, recursive=False, force=False): + self._log(INFO, "Removing directory %s" % path) + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + if not self.isdir(path): + if not self.isfile(path): + raise errors.ResourceNotFoundError(path) + raise errors.ResourceInvalidError(path) + if not force and self.listdir(path): + raise errors.DirectoryNotEmptyError(path) + + self.tahoeutil.unlink(self.dircap, path) + + if recursive and path != '/': + try: + self.removedir(dirname(path), recursive=True) + except errors.DirectoryNotEmptyError: + pass + + @_fix_path + def makedir(self, path, recursive=False, allow_recreate=False): + self._log(INFO, "Creating directory %s" % path) + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + if self.exists(path): + if not self.isdir(path): + raise errors.ResourceInvalidError(path) + if not allow_recreate: + raise errors.DestinationExistsError(path) + if not recursive and not self.exists(dirname(path)): + raise errors.ParentDirectoryMissingError(path) + self.tahoeutil.mkdir(self.dircap, path) + + def movedir(self, src, dst, overwrite=False): + self.move(src, dst, overwrite=overwrite) + + def move(self, src, dst, overwrite=False): + self._log(INFO, "Moving file from %s to %s" % (src, dst)) + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + src = _fixpath(src) + dst = _fixpath(dst) + if not self.exists(dirname(dst)): + raise errors.ParentDirectoryMissingError(dst) + if not overwrite and self.exists(dst): + raise errors.DestinationExistsError(dst) + self.tahoeutil.move(self.dircap, src, dst) + + def rename(self, src, dst): + self.move(src, dst) + + def copy(self, src, dst, overwrite=False, chunk_size=16384): + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + # FIXME: this is out of date; how to do native tahoe copy? + # FIXME: Workaround because isfile() not exists on _TahoeLAFS + FS.copy(self, src, dst, overwrite, chunk_size) + + def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384): + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + # FIXME: this is out of date; how to do native tahoe copy? + # FIXME: Workaround because isfile() not exists on _TahoeLAFS + FS.copydir(self, src, dst, overwrite, ignore_errors, chunk_size) + + + def _log(self, level, message): + if not logger.isEnabledFor(level): return + logger.log(level, u'(%d) %s' % (id(self), + unicode(message).encode('ASCII', 'replace'))) + + @_fix_path + def getpathurl(self, path, allow_none=False, webapi=None): + ''' + Retrieve URL where the file/directory is stored + ''' + if webapi == None: + webapi = self.connection.webapi + self._log(DEBUG, "Retrieving URL for %s over %s" % (path, webapi)) + path = self.tahoeutil.fixwinpath(path, False) + return u"%s/uri/%s%s" % (webapi, self.dircap, path) + + @_fix_path + def getrange(self, path, offset, length=None): + return self.connection.get(u'/uri/%s%s' % (self.dircap, path), + offset=offset, length=length) + + @_fix_path + def setcontents(self, path, file, chunk_size=64*1024): + self._log(INFO, 'Uploading file %s' % path) + size=None + + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + + # Workaround for large files: + # First create zero file placeholder, then + # upload final content. + if self.largefilesize != None and getattr(file, 'read', None): + # As 'file' can be also a string, need to check, + # if 'file' looks like duck. Sorry, file. + file.seek(0, SEEK_END) + size = file.tell() + file.seek(0) + + if size > self.largefilesize: + self.connection.put(u'/uri/%s%s' % (self.dircap, path), + "PyFilesystem.TahoeLAFS: Upload started, final size %d" % size) + + self.connection.put(u'/uri/%s%s' % (self.dircap, path), file, size=size) + + @_fix_path + def getinfo(self, path): + self._log(INFO, 'Reading meta for %s' % path) + info = self.tahoeutil.info(self.dircap, path) + #import datetime + #info['created_time'] = datetime.datetime.now() + #info['modified_time'] = datetime.datetime.now() + #info['accessed_time'] = datetime.datetime.now() + if info['type'] == 'filenode': + info["st_mode"] = 0x700 | statinfo.S_IFREG + elif info['type'] == 'dirnode': + info["st_mode"] = 0x700 | statinfo.S_IFDIR + return info + + + +class TahoeLAFS(CacheFSMixin,_TahoeLAFS): + """FS providing cached access to a Tahoe Filesystem. + + This class is the preferred means to access a Tahoe filesystem. It + maintains an internal cache of recently-accessed metadata to speed + up operations. + """ + + def __init__(self, *args, **kwds): + kwds.setdefault("cache_timeout",60) + super(TahoeLAFS,self).__init__(*args,**kwds) + + diff --git a/fs/contrib/tahoelafs/connection.py b/fs/contrib/tahoelafs/connection.py index e3df28a..448919b 100644 --- a/fs/contrib/tahoelafs/connection.py +++ b/fs/contrib/tahoelafs/connection.py @@ -17,12 +17,12 @@ class PutRequest(Request): def __init__(self, *args, **kwargs): self.get_method = lambda: u'PUT' Request.__init__(self, *args, **kwargs) - + class DeleteRequest(Request): def __init__(self, *args, **kwargs): self.get_method = lambda: u'DELETE' Request.__init__(self, *args, **kwargs) - + class Connection: def __init__(self, webapi): self.webapi = webapi @@ -37,13 +37,13 @@ class Connection: size = len(f) elif getattr(f, 'read', None): if size == None: - # When size is already known, skip this + # When size is already known, skip this f.seek(0, SEEK_END) size = f.tell() f.seek(0) else: raise errors.UnsupportedError("Cannot handle type %s" % type(f)) - + headers = {'Content-Length': size} headers.update(self.headers) return headers @@ -59,7 +59,7 @@ class Connection: if params: return u"%s?%s" % (q, self._urlencode(params)) return q - + def _urlopen(self, req): try: return urlopen(req) @@ -74,17 +74,17 @@ class Connection: # Standard not found raise errors.ResourceNotFoundError(e.fp.read()) raise errors.ResourceInvalidError(e.fp.read()) - + def post(self, path, data={}, params={}): data = self._urlencode(data) path = self._quotepath(path, params) req = Request(''.join([self.webapi, path]), data, headers=self.headers) return self._urlopen(req) - + def get(self, path, data={}, offset=None, length=None): data = self._urlencode(data) path = self._quotepath(path) - if data: + if data: path = u'?'.join([path, data]) headers = {} @@ -95,17 +95,17 @@ class Connection: (int(offset), int(offset+length)) else: headers['Range'] = 'bytes=%d-' % int(offset) - + req = Request(''.join([self.webapi, path]), headers=headers) return self._urlopen(req) def put(self, path, data, size=None, params={}): path = self._quotepath(path, params) headers = self._get_headers(data, size=size) - req = PutRequest(''.join([self.webapi, path]), data, headers=headers) + req = PutRequest(''.join([self.webapi, path]), data, headers=headers) return self._urlopen(req) - - def delete(self, path, data={}): + + def delete(self, path, data={}): path = self._quotepath(path) req = DeleteRequest(''.join([self.webapi, path]), data, headers=self.headers) return self._urlopen(req) diff --git a/fs/contrib/tahoelafs/test_tahoelafs.py b/fs/contrib/tahoelafs/test_tahoelafs.py index 61a28e5..5cb6ca7 100644 --- a/fs/contrib/tahoelafs/test_tahoelafs.py +++ b/fs/contrib/tahoelafs/test_tahoelafs.py @@ -1,52 +1,52 @@ -#!/usr/bin/python
-"""
- Test the TahoeLAFS
-
- @author: Marek Palatinus <marek@palatinus.cz>
-"""
-
-import sys
-import logging
-import unittest
-
-from fs.base import FS
-import fs.errors as errors
-from fs.tests import FSTestCases, ThreadingTestCases
-from fs.contrib.tahoelafs import TahoeLAFS, Connection
-
-logging.getLogger().setLevel(logging.DEBUG)
-logging.getLogger('fs.tahoelafs').addHandler(logging.StreamHandler(sys.stdout))
-
-WEBAPI = 'http://insecure.tahoe-lafs.org'
-
-
-# The public grid is too slow for threading testcases, disabling for now...
-class TestTahoeLAFS(unittest.TestCase,FSTestCases):#,ThreadingTestCases):
-
- # Disabled by default because it takes a *really* long time.
- __test__ = False
-
- def setUp(self):
- self.dircap = TahoeLAFS.createdircap(WEBAPI)
- self.fs = TahoeLAFS(self.dircap, cache_timeout=0, webapi=WEBAPI)
-
- def tearDown(self):
- self.fs.close()
-
- def test_dircap(self):
- # Is dircap in correct format?
- self.assert_(self.dircap.startswith('URI:DIR2:') and len(self.dircap) > 50)
-
- def test_concurrent_copydir(self):
- # makedir() on TahoeLAFS is currently not atomic
- pass
-
- def test_makedir_winner(self):
- # makedir() on TahoeLAFS is currently not atomic
- pass
-
- def test_big_file(self):
- pass
-
-if __name__ == '__main__':
- unittest.main()
+#!/usr/bin/python +""" + Test the TahoeLAFS + + @author: Marek Palatinus <marek@palatinus.cz> +""" + +import sys +import logging +import unittest + +from fs.base import FS +import fs.errors as errors +from fs.tests import FSTestCases, ThreadingTestCases +from fs.contrib.tahoelafs import TahoeLAFS, Connection + +logging.getLogger().setLevel(logging.DEBUG) +logging.getLogger('fs.tahoelafs').addHandler(logging.StreamHandler(sys.stdout)) + +WEBAPI = 'http://insecure.tahoe-lafs.org' + + +# The public grid is too slow for threading testcases, disabling for now... +class TestTahoeLAFS(unittest.TestCase,FSTestCases):#,ThreadingTestCases): + + # Disabled by default because it takes a *really* long time. + __test__ = False + + def setUp(self): + self.dircap = TahoeLAFS.createdircap(WEBAPI) + self.fs = TahoeLAFS(self.dircap, cache_timeout=0, webapi=WEBAPI) + + def tearDown(self): + self.fs.close() + + def test_dircap(self): + # Is dircap in correct format? + self.assert_(self.dircap.startswith('URI:DIR2:') and len(self.dircap) > 50) + + def test_concurrent_copydir(self): + # makedir() on TahoeLAFS is currently not atomic + pass + + def test_makedir_winner(self): + # makedir() on TahoeLAFS is currently not atomic + pass + + def test_big_file(self): + pass + +if __name__ == '__main__': + unittest.main() diff --git a/fs/contrib/tahoelafs/util.py b/fs/contrib/tahoelafs/util.py index fda8a2b..9525f6d 100644 --- a/fs/contrib/tahoelafs/util.py +++ b/fs/contrib/tahoelafs/util.py @@ -1,140 +1,140 @@ -'''
-Created on 25.9.2010
-
-@author: marekp
-'''
-
-import sys
-import platform
-import stat as statinfo
-
-import fs.errors as errors
-from fs.path import pathsplit
-try:
- # For non-CPython or older CPython versions.
- # Simplejson also comes with C speedup module which
- # is not in standard CPython >=2.6 library.
- import simplejson as json
-except ImportError:
- try:
- import json
- except ImportError:
- print "simplejson (http://pypi.python.org/pypi/simplejson/) required"
- raise
-
-from .connection import Connection
-
-python3 = int(platform.python_version_tuple()[0]) > 2
-
-if python3:
- from urllib.error import HTTPError
-else:
- from urllib2 import HTTPError
-
-class TahoeUtil:
- def __init__(self, webapi):
- self.connection = Connection(webapi)
-
- def createdircap(self):
- return self.connection.post(u'/uri', params={u't': u'mkdir'}).read()
-
- def unlink(self, dircap, path=None):
- path = self.fixwinpath(path, False)
- self.connection.delete(u'/uri/%s%s' % (dircap, path))
-
- def info(self, dircap, path):
- path = self.fixwinpath(path, False)
- meta = json.load(self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'}))
- return self._info(path, meta)
-
- def fixwinpath(self, path, direction=True):
- '''
- No, Tahoe really does not support file streams...
- This is ugly hack, because it is not Tahoe-specific.
- Should be move to middleware if will be any.
- '''
- if platform.system() != 'Windows':
- return path
-
- if direction and ':' in path:
- path = path.replace(':', '__colon__')
- elif not direction and '__colon__' in path:
- path = path.replace('__colon__', ':')
- return path
-
- def _info(self, path, data):
- if isinstance(data, list):
- type = data[0]
- data = data[1]
- elif isinstance(data, dict):
- type = data['type']
- else:
- raise errors.ResourceInvalidError('Metadata in unknown format!')
-
- if type == 'unknown':
- raise errors.ResourceNotFoundError(path)
-
- info = {'name': unicode(self.fixwinpath(path, True)),
- 'type': type,
- 'size': data.get('size', 0),
- 'ctime': None,
- 'uri': data.get('rw_uri', data.get('ro_uri'))}
- if 'metadata' in data:
- info['ctime'] = data['metadata'].get('ctime')
-
- if info['type'] == 'dirnode':
- info['st_mode'] = 0777 | statinfo.S_IFDIR
- else:
- info['st_mode'] = 0644
-
- return info
-
- def list(self, dircap, path=None):
- path = self.fixwinpath(path, False)
-
- data = json.load(self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'}))
-
- if len(data) < 2 or data[0] != 'dirnode':
- raise errors.ResourceInvalidError('Metadata in unknown format!')
-
- data = data[1]['children']
- for i in data.keys():
- x = self._info(i, data[i])
- yield x
-
- def mkdir(self, dircap, path):
- path = self.fixwinpath(path, False)
- path = pathsplit(path)
-
- self.connection.post(u"/uri/%s%s" % (dircap, path[0]), data={u't': u'mkdir', u'name': path[1]})
-
- def move(self, dircap, src, dst):
- if src == '/' or dst == '/':
- raise errors.UnsupportedError("Too dangerous operation, aborting")
-
- src = self.fixwinpath(src, False)
- dst = self.fixwinpath(dst, False)
-
- src_tuple = pathsplit(src)
- dst_tuple = pathsplit(dst)
-
- if src_tuple[0] == dst_tuple[0]:
- # Move inside one directory
- self.connection.post(u"/uri/%s%s" % (dircap, src_tuple[0]), data={u't': u'rename',
- u'from_name': src_tuple[1], u'to_name': dst_tuple[1]})
- return
-
- # Move to different directory. Firstly create link on dst, then remove from src
- try:
- self.info(dircap, dst)
- except errors.ResourceNotFoundError:
- pass
- else:
- self.unlink(dircap, dst)
-
- uri = self.info(dircap, src)['uri']
- self.connection.put(u"/uri/%s%s" % (dircap, dst), data=uri, params={u't': u'uri'})
- if uri != self.info(dircap, dst)['uri']:
- raise errors.OperationFailedError('Move failed')
-
- self.unlink(dircap, src)
+''' +Created on 25.9.2010 + +@author: marekp +''' + +import sys +import platform +import stat as statinfo + +import fs.errors as errors +from fs.path import pathsplit +try: + # For non-CPython or older CPython versions. + # Simplejson also comes with C speedup module which + # is not in standard CPython >=2.6 library. + import simplejson as json +except ImportError: + try: + import json + except ImportError: + print "simplejson (http://pypi.python.org/pypi/simplejson/) required" + raise + +from .connection import Connection + +python3 = int(platform.python_version_tuple()[0]) > 2 + +if python3: + from urllib.error import HTTPError +else: + from urllib2 import HTTPError + +class TahoeUtil: + def __init__(self, webapi): + self.connection = Connection(webapi) + + def createdircap(self): + return self.connection.post(u'/uri', params={u't': u'mkdir'}).read() + + def unlink(self, dircap, path=None): + path = self.fixwinpath(path, False) + self.connection.delete(u'/uri/%s%s' % (dircap, path)) + + def info(self, dircap, path): + path = self.fixwinpath(path, False) + meta = json.load(self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'})) + return self._info(path, meta) + + def fixwinpath(self, path, direction=True): + ''' + No, Tahoe really does not support file streams... + This is ugly hack, because it is not Tahoe-specific. + Should be move to middleware if will be any. + ''' + if platform.system() != 'Windows': + return path + + if direction and ':' in path: + path = path.replace(':', '__colon__') + elif not direction and '__colon__' in path: + path = path.replace('__colon__', ':') + return path + + def _info(self, path, data): + if isinstance(data, list): + type = data[0] + data = data[1] + elif isinstance(data, dict): + type = data['type'] + else: + raise errors.ResourceInvalidError('Metadata in unknown format!') + + if type == 'unknown': + raise errors.ResourceNotFoundError(path) + + info = {'name': unicode(self.fixwinpath(path, True)), + 'type': type, + 'size': data.get('size', 0), + 'ctime': None, + 'uri': data.get('rw_uri', data.get('ro_uri'))} + if 'metadata' in data: + info['ctime'] = data['metadata'].get('ctime') + + if info['type'] == 'dirnode': + info['st_mode'] = 0777 | statinfo.S_IFDIR + else: + info['st_mode'] = 0644 + + return info + + def list(self, dircap, path=None): + path = self.fixwinpath(path, False) + + data = json.load(self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'})) + + if len(data) < 2 or data[0] != 'dirnode': + raise errors.ResourceInvalidError('Metadata in unknown format!') + + data = data[1]['children'] + for i in data.keys(): + x = self._info(i, data[i]) + yield x + + def mkdir(self, dircap, path): + path = self.fixwinpath(path, False) + path = pathsplit(path) + + self.connection.post(u"/uri/%s%s" % (dircap, path[0]), data={u't': u'mkdir', u'name': path[1]}) + + def move(self, dircap, src, dst): + if src == '/' or dst == '/': + raise errors.UnsupportedError("Too dangerous operation, aborting") + + src = self.fixwinpath(src, False) + dst = self.fixwinpath(dst, False) + + src_tuple = pathsplit(src) + dst_tuple = pathsplit(dst) + + if src_tuple[0] == dst_tuple[0]: + # Move inside one directory + self.connection.post(u"/uri/%s%s" % (dircap, src_tuple[0]), data={u't': u'rename', + u'from_name': src_tuple[1], u'to_name': dst_tuple[1]}) + return + + # Move to different directory. Firstly create link on dst, then remove from src + try: + self.info(dircap, dst) + except errors.ResourceNotFoundError: + pass + else: + self.unlink(dircap, dst) + + uri = self.info(dircap, src)['uri'] + self.connection.put(u"/uri/%s%s" % (dircap, dst), data=uri, params={u't': u'uri'}) + if uri != self.info(dircap, dst)['uri']: + raise errors.OperationFailedError('Move failed') + + self.unlink(dircap, src) |