Rename glance.store to glance_store0.1.1

Given the existing, known issues, with python namespaces, pip and
setuptools, we've decided to rename glance.store into glance_store.

Change-Id: I3f02ce78b3d64f34744e5116652bfbb4f3062cbf

4 files changed, 647 insertions, 0 deletions

diff --git a/glance_store/common/__init__.py b/glance_store/common/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/glance_store/common/__init__.py
diff --git a/glance_store/common/auth.py b/glance_store/common/auth.py
new file mode 100644
index 0000000..2dffd15
--- /dev/null
+++ b/glance_store/common/auth.py
@@ -0,0 +1,288 @@
+# Copyright 2011 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+This auth module is intended to allow OpenStack client-tools to select from a
+variety of authentication strategies, including NoAuth (the default), and
+Keystone (an identity management system).
+
+    > auth_plugin = AuthPlugin(creds)
+
+    > auth_plugin.authenticate()
+
+    > auth_plugin.auth_token
+    abcdefg
+
+    > auth_plugin.management_url
+    http://service_endpoint/
+"""
+import httplib2
+import logging
+
+import six.moves.urllib.parse as urlparse
+
+from glance_store.openstack.common import jsonutils
+
+
+LOG = logging.getLogger(__name__)
+
+
+class BaseStrategy(object):
+    def __init__(self):
+        self.auth_token = None
+        # TODO(sirp): Should expose selecting public/internal/admin URL.
+        self.management_url = None
+
+    def authenticate(self):
+        raise NotImplementedError
+
+    @property
+    def is_authenticated(self):
+        raise NotImplementedError
+
+    @property
+    def strategy(self):
+        raise NotImplementedError
+
+
+class NoAuthStrategy(BaseStrategy):
+    def authenticate(self):
+        pass
+
+    @property
+    def is_authenticated(self):
+        return True
+
+    @property
+    def strategy(self):
+        return 'noauth'
+
+
+class KeystoneStrategy(BaseStrategy):
+    MAX_REDIRECTS = 10
+
+    def __init__(self, creds, insecure=False, configure_via_auth=True):
+        self.creds = creds
+        self.insecure = insecure
+        self.configure_via_auth = configure_via_auth
+        super(KeystoneStrategy, self).__init__()
+
+    def check_auth_params(self):
+        # Ensure that supplied credential parameters are as required
+        for required in ('username', 'password', 'auth_url',
+                         'strategy'):
+            if self.creds.get(required) is None:
+                raise exception.MissingCredentialError(required=required)
+        if self.creds['strategy'] != 'keystone':
+            raise exception.BadAuthStrategy(expected='keystone',
+                                            received=self.creds['strategy'])
+        # For v2.0 also check tenant is present
+        if self.creds['auth_url'].rstrip('/').endswith('v2.0'):
+            if self.creds.get("tenant") is None:
+                raise exception.MissingCredentialError(required='tenant')
+
+    def authenticate(self):
+        """Authenticate with the Keystone service.
+
+        There are a few scenarios to consider here:
+
+        1. Which version of Keystone are we using? v1 which uses headers to
+           pass the credentials, or v2 which uses a JSON encoded request body?
+
+        2. Keystone may respond back with a redirection using a 305 status
+           code.
+
+        3. We may attempt a v1 auth when v2 is what's called for. In this
+           case, we rewrite the url to contain /v2.0/ and retry using the v2
+           protocol.
+        """
+        def _authenticate(auth_url):
+            # If OS_AUTH_URL is missing a trailing slash add one
+            if not auth_url.endswith('/'):
+                auth_url += '/'
+            token_url = urlparse.urljoin(auth_url, "tokens")
+            # 1. Check Keystone version
+            is_v2 = auth_url.rstrip('/').endswith('v2.0')
+            if is_v2:
+                self._v2_auth(token_url)
+            else:
+                self._v1_auth(token_url)
+
+        self.check_auth_params()
+        auth_url = self.creds['auth_url']
+        for _ in range(self.MAX_REDIRECTS):
+            try:
+                _authenticate(auth_url)
+            except exception.AuthorizationRedirect as e:
+                # 2. Keystone may redirect us
+                auth_url = e.url
+            except exception.AuthorizationFailure:
+                # 3. In some configurations nova makes redirection to
+                # v2.0 keystone endpoint. Also, new location does not
+                # contain real endpoint, only hostname and port.
+                if 'v2.0' not in auth_url:
+                    auth_url = urlparse.urljoin(auth_url, 'v2.0/')
+            else:
+                # If we successfully auth'd, then memorize the correct auth_url
+                # for future use.
+                self.creds['auth_url'] = auth_url
+                break
+        else:
+            # Guard against a redirection loop
+            raise exception.MaxRedirectsExceeded(redirects=self.MAX_REDIRECTS)
+
+    def _v1_auth(self, token_url):
+        creds = self.creds
+
+        headers = {}
+        headers['X-Auth-User'] = creds['username']
+        headers['X-Auth-Key'] = creds['password']
+
+        tenant = creds.get('tenant')
+        if tenant:
+            headers['X-Auth-Tenant'] = tenant
+
+        resp, resp_body = self._do_request(token_url, 'GET', headers=headers)
+
+        def _management_url(self, resp):
+            for url_header in ('x-image-management-url',
+                               'x-server-management-url',
+                               'x-glance'):
+                try:
+                    return resp[url_header]
+                except KeyError as e:
+                    not_found = e
+            raise not_found
+
+        if resp.status in (200, 204):
+            try:
+                if self.configure_via_auth:
+                    self.management_url = _management_url(self, resp)
+                self.auth_token = resp['x-auth-token']
+            except KeyError:
+                raise exception.AuthorizationFailure()
+        elif resp.status == 305:
+            raise exception.AuthorizationRedirect(uri=resp['location'])
+        elif resp.status == 400:
+            raise exception.AuthBadRequest(url=token_url)
+        elif resp.status == 401:
+            raise exception.NotAuthenticated()
+        elif resp.status == 404:
+            raise exception.AuthUrlNotFound(url=token_url)
+        else:
+            raise Exception(_('Unexpected response: %s') % resp.status)
+
+    def _v2_auth(self, token_url):
+
+        creds = self.creds
+
+        creds = {
+            "auth": {
+                "tenantName": creds['tenant'],
+                "passwordCredentials": {
+                    "username": creds['username'],
+                    "password": creds['password']
+                }
+            }
+        }
+
+        headers = {}
+        headers['Content-Type'] = 'application/json'
+        req_body = jsonutils.dumps(creds)
+
+        resp, resp_body = self._do_request(
+            token_url, 'POST', headers=headers, body=req_body)
+
+        if resp.status == 200:
+            resp_auth = jsonutils.loads(resp_body)['access']
+            creds_region = self.creds.get('region')
+            if self.configure_via_auth:
+                endpoint = get_endpoint(resp_auth['serviceCatalog'],
+                                        endpoint_region=creds_region)
+                self.management_url = endpoint
+            self.auth_token = resp_auth['token']['id']
+        elif resp.status == 305:
+            raise exception.RedirectException(resp['location'])
+        elif resp.status == 400:
+            raise exception.AuthBadRequest(url=token_url)
+        elif resp.status == 401:
+            raise exception.NotAuthenticated()
+        elif resp.status == 404:
+            raise exception.AuthUrlNotFound(url=token_url)
+        else:
+            raise Exception(_('Unexpected response: %s') % resp.status)
+
+    @property
+    def is_authenticated(self):
+        return self.auth_token is not None
+
+    @property
+    def strategy(self):
+        return 'keystone'
+
+    def _do_request(self, url, method, headers=None, body=None):
+        headers = headers or {}
+        conn = httplib2.Http()
+        conn.force_exception_to_status_code = True
+        conn.disable_ssl_certificate_validation = self.insecure
+        headers['User-Agent'] = 'glance-client'
+        resp, resp_body = conn.request(url, method, headers=headers, body=body)
+        return resp, resp_body
+
+
+def get_plugin_from_strategy(strategy, creds=None, insecure=False,
+                             configure_via_auth=True):
+    if strategy == 'noauth':
+        return NoAuthStrategy()
+    elif strategy == 'keystone':
+        return KeystoneStrategy(creds, insecure,
+                                configure_via_auth=configure_via_auth)
+    else:
+        raise Exception(_("Unknown auth strategy '%s'") % strategy)
+
+
+def get_endpoint(service_catalog, service_type='image', endpoint_region=None,
+                 endpoint_type='publicURL'):
+    """
+    Select an endpoint from the service catalog
+
+    We search the full service catalog for services
+    matching both type and region. If the client
+    supplied no region then any 'image' endpoint
+    is considered a match. There must be one -- and
+    only one -- successful match in the catalog,
+    otherwise we will raise an exception.
+    """
+    endpoint = None
+    for service in service_catalog:
+        s_type = None
+        try:
+            s_type = service['type']
+        except KeyError:
+            msg = _('Encountered service with no "type": %s') % s_type
+            LOG.warn(msg)
+            continue
+
+        if s_type == service_type:
+            for ep in service['endpoints']:
+                if endpoint_region is None or endpoint_region == ep['region']:
+                    if endpoint is not None:
+                        # This is a second match, abort
+                        raise exception.RegionAmbiguity(region=endpoint_region)
+                    endpoint = ep
+    if endpoint and endpoint.get(endpoint_type):
+        return endpoint[endpoint_type]
+    else:
+        raise exception.NoServiceEndpoint()
diff --git a/glance_store/common/config.py b/glance_store/common/config.py
new file mode 100644
index 0000000..3177b07
--- /dev/null
+++ b/glance_store/common/config.py
@@ -0,0 +1,220 @@
+#!/usr/bin/env python
+
+# Copyright 2011 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Routines for configuring Glance
+"""
+
+import logging
+import logging.config
+import logging.handlers
+import os
+
+from oslo.config import cfg
+from paste import deploy
+
+from glance.version import version_info as version
+
+paste_deploy_opts = [
+    cfg.StrOpt('flavor',
+               help=_('Partial name of a pipeline in your paste configuration '
+                      'file with the service name removed. For example, if '
+                      'your paste section name is '
+                      '[pipeline:glance-api-keystone] use the value '
+                      '"keystone"')),
+    cfg.StrOpt('config_file',
+               help=_('Name of the paste configuration file.')),
+]
+image_format_opts = [
+    cfg.ListOpt('container_formats',
+                default=['ami', 'ari', 'aki', 'bare', 'ovf'],
+                help=_("Supported values for the 'container_format' "
+                       "image attribute"),
+                deprecated_opts=[cfg.DeprecatedOpt('container_formats',
+                                                   group='DEFAULT')]),
+    cfg.ListOpt('disk_formats',
+                default=['ami', 'ari', 'aki', 'vhd', 'vmdk', 'raw', 'qcow2',
+                         'vdi', 'iso'],
+                help=_("Supported values for the 'disk_format' "
+                       "image attribute"),
+                deprecated_opts=[cfg.DeprecatedOpt('disk_formats',
+                                                   group='DEFAULT')]),
+]
+task_opts = [
+    cfg.IntOpt('task_time_to_live',
+               default=48,
+               help=_("Time in hours for which a task lives after, either "
+                      "succeeding or failing"),
+               deprecated_opts=[cfg.DeprecatedOpt('task_time_to_live',
+                                                  group='DEFAULT')]),
+]
+common_opts = [
+    cfg.BoolOpt('allow_additional_image_properties', default=True,
+                help=_('Whether to allow users to specify image properties '
+                       'beyond what the image schema provides')),
+    cfg.IntOpt('image_member_quota', default=128,
+               help=_('Maximum number of image members per image. '
+                      'Negative values evaluate to unlimited.')),
+    cfg.IntOpt('image_property_quota', default=128,
+               help=_('Maximum number of properties allowed on an image. '
+                      'Negative values evaluate to unlimited.')),
+    cfg.IntOpt('image_tag_quota', default=128,
+               help=_('Maximum number of tags allowed on an image. '
+                      'Negative values evaluate to unlimited.')),
+    cfg.IntOpt('image_location_quota', default=10,
+               help=_('Maximum number of locations allowed on an image. '
+                      'Negative values evaluate to unlimited.')),
+    cfg.StrOpt('data_api', default='glance.db.sqlalchemy.api',
+               help=_('Python module path of data access API')),
+    cfg.IntOpt('limit_param_default', default=25,
+               help=_('Default value for the number of items returned by a '
+                      'request if not specified explicitly in the request')),
+    cfg.IntOpt('api_limit_max', default=1000,
+               help=_('Maximum permissible number of items that could be '
+                      'returned by a request')),
+    cfg.BoolOpt('show_image_direct_url', default=False,
+                help=_('Whether to include the backend image storage location '
+                       'in image properties. Revealing storage location can '
+                       'be a security risk, so use this setting with '
+                       'caution!')),
+    cfg.BoolOpt('show_multiple_locations', default=False,
+                help=_('Whether to include the backend image locations '
+                       'in image properties. Revealing storage location can '
+                       'be a security risk, so use this setting with '
+                       'caution!  The overrides show_image_direct_url.')),
+    cfg.IntOpt('image_size_cap', default=1099511627776,
+               help=_("Maximum size of image a user can upload in bytes. "
+                      "Defaults to 1099511627776 bytes (1 TB).")),
+    cfg.IntOpt('user_storage_quota', default=0,
+               help=_("Set a system wide quota for every user.  This value is "
+                      "the total number of bytes that a user can use across "
+                      "all storage systems.  A value of 0 means unlimited.")),
+    cfg.BoolOpt('enable_v1_api', default=True,
+                help=_("Deploy the v1 OpenStack Images API.")),
+    cfg.BoolOpt('enable_v2_api', default=True,
+                help=_("Deploy the v2 OpenStack Images API.")),
+    cfg.StrOpt('pydev_worker_debug_host', default=None,
+               help=_('The hostname/IP of the pydev process listening for '
+                      'debug connections')),
+    cfg.IntOpt('pydev_worker_debug_port', default=5678,
+               help=_('The port on which a pydev process is listening for '
+                      'connections.')),
+    cfg.StrOpt('metadata_encryption_key', secret=True,
+               help=_('Key used for encrypting sensitive metadata while '
+                      'talking to the registry or database.')),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(paste_deploy_opts, group='paste_deploy')
+CONF.register_opts(image_format_opts, group='image_format')
+CONF.register_opts(task_opts, group='task')
+CONF.register_opts(common_opts)
+
+
+def parse_args(args=None, usage=None, default_config_files=None):
+    CONF(args=args,
+         project='glance',
+         version=version.cached_version_string(),
+         usage=usage,
+         default_config_files=default_config_files)
+
+
+def parse_cache_args(args=None):
+    config_files = cfg.find_config_files(project='glance', prog='glance-cache')
+    parse_args(args=args, default_config_files=config_files)
+
+
+def _get_deployment_flavor(flavor=None):
+    """
+    Retrieve the paste_deploy.flavor config item, formatted appropriately
+    for appending to the application name.
+
+    :param flavor: if specified, use this setting rather than the
+                   paste_deploy.flavor configuration setting
+    """
+    if not flavor:
+        flavor = CONF.paste_deploy.flavor
+    return '' if not flavor else ('-' + flavor)
+
+
+def _get_paste_config_path():
+    paste_suffix = '-paste.ini'
+    conf_suffix = '.conf'
+    if CONF.config_file:
+        # Assume paste config is in a paste.ini file corresponding
+        # to the last config file
+        path = CONF.config_file[-1].replace(conf_suffix, paste_suffix)
+    else:
+        path = CONF.prog + paste_suffix
+    return CONF.find_file(os.path.basename(path))
+
+
+def _get_deployment_config_file():
+    """
+    Retrieve the deployment_config_file config item, formatted as an
+    absolute pathname.
+    """
+    path = CONF.paste_deploy.config_file
+    if not path:
+        path = _get_paste_config_path()
+    if not path:
+        msg = "Unable to locate paste config file for %s." % CONF.prog
+        raise RuntimeError(msg)
+    return os.path.abspath(path)
+
+
+def load_paste_app(app_name, flavor=None, conf_file=None):
+    """
+    Builds and returns a WSGI app from a paste config file.
+
+    We assume the last config file specified in the supplied ConfigOpts
+    object is the paste config file, if conf_file is None.
+
+    :param app_name: name of the application to load
+    :param flavor: name of the variant of the application to load
+    :param conf_file: path to the paste config file
+
+    :raises RuntimeError when config file cannot be located or application
+            cannot be loaded from config file
+    """
+    # append the deployment flavor to the application name,
+    # in order to identify the appropriate paste pipeline
+    app_name += _get_deployment_flavor(flavor)
+
+    if not conf_file:
+        conf_file = _get_deployment_config_file()
+
+    try:
+        logger = logging.getLogger(__name__)
+        logger.debug(_("Loading %(app_name)s from %(conf_file)s"),
+                     {'conf_file': conf_file, 'app_name': app_name})
+
+        app = deploy.loadapp("config:%s" % conf_file, name=app_name)
+
+        # Log the options used when starting if we're in debug mode...
+        if CONF.debug:
+            CONF.log_opt_values(logger, logging.DEBUG)
+
+        return app
+    except (LookupError, ImportError) as e:
+        msg = (_("Unable to load %(app_name)s from "
+                 "configuration file %(conf_file)s."
+                 "\nGot: %(e)r") % {'app_name': app_name,
+                                    'conf_file': conf_file,
+                                    'e': e})
+        logger.error(msg)
+        raise RuntimeError(msg)
diff --git a/glance_store/common/utils.py b/glance_store/common/utils.py
new file mode 100644
index 0000000..fb77519
--- /dev/null
+++ b/glance_store/common/utils.py
@@ -0,0 +1,139 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+System-level utilities and helper functions.
+"""
+
+import logging
+import uuid
+
+try:
+    from eventlet import sleep
+except ImportError:
+    from time import sleep
+
+
+LOG = logging.getLogger(__name__)
+
+
+def is_uuid_like(val):
+    """Returns validation of a value as a UUID.
+
+    For our purposes, a UUID is a canonical form string:
+    aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
+    """
+
+    try:
+        return str(uuid.UUID(val)) == val
+    except (TypeError, ValueError, AttributeError):
+        return False
+
+
+def chunkreadable(iter, chunk_size=65536):
+    """
+    Wrap a readable iterator with a reader yielding chunks of
+    a preferred size, otherwise leave iterator unchanged.
+
+    :param iter: an iter which may also be readable
+    :param chunk_size: maximum size of chunk
+    """
+    return chunkiter(iter, chunk_size) if hasattr(iter, 'read') else iter
+
+
+def chunkiter(fp, chunk_size=65536):
+    """
+    Return an iterator to a file-like obj which yields fixed size chunks
+
+    :param fp: a file-like object
+    :param chunk_size: maximum size of chunk
+    """
+    while True:
+        chunk = fp.read(chunk_size)
+        if chunk:
+            yield chunk
+        else:
+            break
+
+
+def cooperative_iter(iter):
+    """
+    Return an iterator which schedules after each
+    iteration. This can prevent eventlet thread starvation.
+
+    :param iter: an iterator to wrap
+    """
+    try:
+        for chunk in iter:
+            sleep(0)
+            yield chunk
+    except Exception as err:
+        msg = _("Error: cooperative_iter exception %s") % err
+        LOG.error(msg)
+        raise
+
+
+def cooperative_read(fd):
+    """
+    Wrap a file descriptor's read with a partial function which schedules
+    after each read. This can prevent eventlet thread starvation.
+
+    :param fd: a file descriptor to wrap
+    """
+    def readfn(*args):
+        result = fd.read(*args)
+        sleep(0)
+        return result
+    return readfn
+
+
+class CooperativeReader(object):
+    """
+    An eventlet thread friendly class for reading in image data.
+
+    When accessing data either through the iterator or the read method
+    we perform a sleep to allow a co-operative yield. When there is more than
+    one image being uploaded/downloaded this prevents eventlet thread
+    starvation, ie allows all threads to be scheduled periodically rather than
+    having the same thread be continuously active.
+    """
+    def __init__(self, fd):
+        """
+        :param fd: Underlying image file object
+        """
+        self.fd = fd
+        self.iterator = None
+        # NOTE(markwash): if the underlying supports read(), overwrite the
+        # default iterator-based implementation with cooperative_read which
+        # is more straightforward
+        if hasattr(fd, 'read'):
+            self.read = cooperative_read(fd)
+
+    def read(self, length=None):
+        """Return the next chunk of the underlying iterator.
+
+        This is replaced with cooperative_read in __init__ if the underlying
+        fd already supports read().
+        """
+        if self.iterator is None:
+            self.iterator = self.__iter__()
+        try:
+            return self.iterator.next()
+        except StopIteration:
+            return ''
+
+    def __iter__(self):
+        return cooperative_iter(self.fd.__iter__())