path: root/swift/obj/mem_diskfile.py

# Copyright (c) 2010-2013 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" In-Memory Disk File Interface for Swift Object Server"""

import io
import time
from contextlib import contextmanager

from eventlet import Timeout

from swift.common.utils import Timestamp
from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
    DiskFileCollision, DiskFileDeleted, DiskFileNotOpen
from swift.common.request_helpers import is_sys_meta
from swift.common.swob import multi_range_iterator
from swift.common.utils import md5
from swift.obj.diskfile import DATAFILE_SYSTEM_META, RESERVED_DATAFILE_META


class InMemoryFileSystem(object):
    """
    A very simplistic in-memory file system scheme.

    There is one dictionary mapping a given object name to a tuple. The first
    entry in the tuple is the BytesIO buffer representing the file contents,
    the second entry is the metadata dictionary.
    """

    def __init__(self):
        self._filesystem = {}

    def get_object(self, name):
        """
        Return back an file-like object and its metadata

        :param name: standard object name
        :return: (fp, metadata) fp is ``BytesIO`` in-memory representation
                                object (or None). metadata is a dictionary
                                of metadata (or None)
        """
        val = self._filesystem.get(name)
        if val is None:
            fp, metadata = None, None
        else:
            fp, metadata = val
        return fp, metadata

    def put_object(self, name, fp, metadata):
        """
        Store object into memory

        :param name: standard object name
        :param fp: ``BytesIO`` in-memory representation object
        :param metadata: dictionary of metadata to be written
        """
        self._filesystem[name] = (fp, metadata)

    def del_object(self, name):
        """
        Delete object from memory

        :param name: standard object name
        """
        del self._filesystem[name]

    def get_diskfile(self, account, container, obj, **kwargs):
        return DiskFile(self, account, container, obj)

    def pickle_async_update(self, *args, **kwargs):
        """
        For now don't handle async updates.
        """
        pass


class DiskFileWriter(object):
    """
    .. note::
        Sample alternative pluggable on-disk backend implementation.

    Encapsulation of the write context for servicing PUT REST API
    requests. Serves as the context manager object for DiskFile's create()
    method.

    :param fs: internal file system object to use
    :param name: standard object name
    """
    def __init__(self, fs, name):
        self._filesystem = fs
        self._name = name
        self._fp = None
        self._upload_size = 0
        self._chunks_etag = md5(usedforsecurity=False)

    def open(self):
        """
        Prepare to accept writes.

        Create a new ``BytesIO`` object for a started-but-not-yet-finished
        PUT.
        """
        self._fp = io.BytesIO()
        return self

    def close(self):
        """
        Clean up resources following an ``open()``.

        Note: If ``put()`` has not been called, the data written will be lost.
        """
        self._fp = None

    def write(self, chunk):
        """
        Write a chunk of data into the ``BytesIO`` object.

        :param chunk: the chunk of data to write as a string object
        """
        self._fp.write(chunk)
        self._upload_size += len(chunk)
        self._chunks_etag.update(chunk)

    def chunks_finished(self):
        """
        Expose internal stats about written chunks.

        :returns: a tuple, (upload_size, etag)
        """
        return self._upload_size, self._chunks_etag.hexdigest()

    def put(self, metadata):
        """
        Make the final association in the in-memory file system for this name
        with the ``BytesIO`` object.

        :param metadata: dictionary of metadata to be written
        """
        metadata['name'] = self._name
        self._filesystem.put_object(self._name, self._fp, metadata)

    def commit(self, timestamp):
        """
        Perform any operations necessary to mark the object as durable. For
        mem_diskfile type this is a no-op.

        :param timestamp: object put timestamp, an instance of
                          :class:`~swift.common.utils.Timestamp`
        """
        pass


class DiskFileReader(object):
    """
    .. note::
        Sample alternative pluggable on-disk backend implementation.

    Encapsulation of the read context for servicing GET REST API
    requests. Serves as the context manager object for DiskFile's reader()
    method.

    :param name: object name
    :param fp: open file object pointer reference
    :param obj_size: on-disk size of object in bytes
    :param etag: MD5 hash of object from metadata
    """
    def __init__(self, name, fp, obj_size, etag):
        self._name = name
        self._fp = fp
        self._obj_size = obj_size
        self._etag = etag
        #
        self._iter_etag = None
        self._bytes_read = 0
        self._started_at_0 = False
        self._read_to_eof = False
        self._suppress_file_closing = False
        #
        self.was_quarantined = ''

    def __iter__(self):
        try:
            self._bytes_read = 0
            self._started_at_0 = False
            self._read_to_eof = False
            if self._fp.tell() == 0:
                self._started_at_0 = True
                self._iter_etag = md5(usedforsecurity=False)
            while True:
                chunk = self._fp.read()
                if chunk:
                    if self._iter_etag:
                        self._iter_etag.update(chunk)
                    self._bytes_read += len(chunk)
                    yield chunk
                else:
                    self._read_to_eof = True
                    break
        finally:
            if not self._suppress_file_closing:
                self.close()

    def app_iter_range(self, start, stop):
        if start or start == 0:
            self._fp.seek(start)
        if stop is not None:
            length = stop - start
        else:
            length = None
        try:
            for chunk in self:
                if length is not None:
                    length -= len(chunk)
                    if length < 0:
                        # Chop off the extra:
                        yield chunk[:length]
                        break
                yield chunk
        finally:
            if not self._suppress_file_closing:
                self.close()

    def app_iter_ranges(self, ranges, content_type, boundary, size):
        if not ranges:
            yield ''
        else:
            try:
                self._suppress_file_closing = True
                for chunk in multi_range_iterator(
                        ranges, content_type, boundary, size,
                        self.app_iter_range):
                    yield chunk
            finally:
                self._suppress_file_closing = False
                try:
                    self.close()
                except DiskFileQuarantined:
                    pass

    def _quarantine(self, msg):
        self.was_quarantined = msg

    def _handle_close_quarantine(self):
        if self._bytes_read != self._obj_size:
            self._quarantine(
                "Bytes read: %s, does not match metadata: %s" % (
                    self._bytes_read, self._obj_size))
        elif self._iter_etag and \
                self._etag != self._iter_etag.hexdigest():
            self._quarantine(
                "ETag %s and file's md5 %s do not match" % (
                    self._etag, self._iter_etag.hexdigest()))

    def close(self):
        """
        Close the file. Will handle quarantining file if necessary.
        """
        if self._fp:
            try:
                if self._started_at_0 and self._read_to_eof:
                    self._handle_close_quarantine()
            except (Exception, Timeout):
                pass
            finally:
                self._fp = None


class DiskFile(object):
    """
    .. note::

        Sample alternative pluggable on-disk backend implementation. This
        example duck-types the reference implementation DiskFile class.

    Manage object files in-memory.

    :param fs: an instance of InMemoryFileSystem
    :param account: account name for the object
    :param container: container name for the object
    :param obj: object name for the object
    """

    def __init__(self, fs, account, container, obj):
        self._name = '/' + '/'.join((account, container, obj))
        self._metadata = None
        self._fp = None
        self._filesystem = fs
        self.fragments = None

    def open(self, modernize=False, current_time=None):
        """
        Open the file and read the metadata.

        This method must populate the _metadata attribute.

        :param current_time: Unix time used in checking expiration. If not
             present, the current time will be used.
        :raises DiskFileCollision: on name mis-match with metadata
        :raises DiskFileDeleted: if it does not exist, or a tombstone is
                                 present
        :raises DiskFileQuarantined: if while reading metadata of the file
                                     some data did pass cross checks
        """
        fp, self._metadata = self._filesystem.get_object(self._name)
        if fp is None:
            raise DiskFileDeleted()
        self._fp = self._verify_data_file(fp, current_time)
        self._metadata = self._metadata or {}
        return self

    def __enter__(self):
        if self._metadata is None:
            raise DiskFileNotOpen()
        return self

    def __exit__(self, t, v, tb):
        if self._fp is not None:
            self._fp = None

    def _quarantine(self, name, msg):
        """
        Quarantine a file; responsible for incrementing the associated logger's
        count of quarantines.

        :param name: name of object to quarantine
        :param msg: reason for quarantining to be included in the exception
        :returns: DiskFileQuarantined exception object
        """
        # for this implementation we simply delete the bad object
        self._filesystem.del_object(name)
        return DiskFileQuarantined(msg)

    def _verify_data_file(self, fp, current_time):
        """
        Verify the metadata's name value matches what we think the object is
        named.

        :raises DiskFileCollision: if the metadata stored name does not match
                                   the referenced name of the file
        :raises DiskFileNotExist: if the object has expired
        :raises DiskFileQuarantined: if data inconsistencies were detected
                                     between the metadata and the file-system
                                     metadata
        """
        try:
            mname = self._metadata['name']
        except KeyError:
            raise self._quarantine(self._name, "missing name metadata")
        else:
            if mname != self._name:
                raise DiskFileCollision('Client path does not match path '
                                        'stored in object metadata')
        try:
            x_delete_at = int(self._metadata['X-Delete-At'])
        except KeyError:
            pass
        except ValueError:
            # Quarantine, the x-delete-at key is present but not an
            # integer.
            raise self._quarantine(
                self._name, "bad metadata x-delete-at value %s" % (
                    self._metadata['X-Delete-At']))
        else:
            if current_time is None:
                current_time = time.time()
            if x_delete_at <= current_time:
                raise DiskFileNotExist('Expired')
        try:
            metadata_size = int(self._metadata['Content-Length'])
        except KeyError:
            raise self._quarantine(
                self._name, "missing content-length in metadata")
        except ValueError:
            # Quarantine, the content-length key is present but not an
            # integer.
            raise self._quarantine(
                self._name, "bad metadata content-length value %s" % (
                    self._metadata['Content-Length']))
        try:
            fp.seek(0, 2)
            obj_size = fp.tell()
            fp.seek(0, 0)
        except OSError as err:
            # Quarantine, we can't successfully stat the file.
            raise self._quarantine(self._name, "not stat-able: %s" % err)
        if obj_size != metadata_size:
            raise self._quarantine(
                self._name, "metadata content-length %s does"
                " not match actual object size %s" % (
                    metadata_size, obj_size))
        return fp

    def get_metadata(self):
        """
        Provide the metadata for an object as a dictionary.

        :returns: object's metadata dictionary
        """
        if self._metadata is None:
            raise DiskFileNotOpen()
        return self._metadata

    get_datafile_metadata = get_metadata
    get_metafile_metadata = get_metadata

    def read_metadata(self, current_time=None):
        """
        Return the metadata for an object.

        :param current_time: Unix time used in checking expiration. If not
             present, the current time will be used.
        :returns: metadata dictionary for an object
        """
        with self.open(current_time=current_time):
            return self.get_metadata()

    def reader(self, keep_cache=False):
        """
        Return a swift.common.swob.Response class compatible "app_iter"
        object. The responsibility of closing the open file is passed to the
        DiskFileReader object.

        :param keep_cache:
        """
        dr = DiskFileReader(self._name, self._fp,
                            int(self._metadata['Content-Length']),
                            self._metadata['ETag'])
        # At this point the reader object is now responsible for
        # the file pointer.
        self._fp = None
        return dr

    def writer(self, size=None):
        return DiskFileWriter(self._filesystem, self._name)

    @contextmanager
    def create(self, size=None):
        """
        Context manager to create a file. We create a temporary file first, and
        then return a DiskFileWriter object to encapsulate the state.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        """
        writer = self.writer(size)
        try:
            yield writer.open()
        finally:
            writer.close()

    def write_metadata(self, metadata):
        """
        Write a block of metadata to an object.
        """
        data, cur_mdata = self._filesystem.get_object(self._name)
        if data is not None:
            # The object exists. Update the new metadata with the object's
            # immutable metadata (e.g. name, size, etag, sysmeta) and store it
            # with the object data.
            immutable_metadata = dict(
                [(key, val) for key, val in cur_mdata.items()
                 if key.lower() in (RESERVED_DATAFILE_META |
                                    DATAFILE_SYSTEM_META)
                 or is_sys_meta('object', key)])
            metadata.update(immutable_metadata)
            metadata['name'] = self._name
            self._filesystem.put_object(self._name, data, metadata)

    def delete(self, timestamp):
        """
        Perform a delete for the given object in the given container under the
        given account.

        This creates a tombstone file with the given timestamp, and removes
        any older versions of the object file.  Any file that has an older
        timestamp than timestamp will be deleted.

        :param timestamp: timestamp to compare with each file
        """
        fp, md = self._filesystem.get_object(self._name)
        if md and md['X-Timestamp'] < Timestamp(timestamp):
            self._filesystem.del_object(self._name)

    @property
    def timestamp(self):
        if self._metadata is None:
            raise DiskFileNotOpen()
        return Timestamp(self._metadata.get('X-Timestamp'))

    data_timestamp = timestamp

    durable_timestamp = timestamp

    content_type_timestamp = timestamp

    @property
    def content_type(self):
        if self._metadata is None:
            raise DiskFileNotOpen()
        return self._metadata.get('Content-Type')