Add location prefix url to store instance

For updating location metadata with store information to the images
which are existed prior to enabling multiple stores, added new
public attribute 'url_prefix' to each of the store instance.

Implements: blueprint location-uri-prefix

Change-Id: Icd760d30e947867c2b5b87f86bbe4b1a4240d214

14 files changed, 159 insertions, 12 deletions

diff --git a/glance_store/_drivers/cinder.py b/glance_store/_drivers/cinder.py
index 0b5ac1d..7212e02 100644
--- a/glance_store/_drivers/cinder.py
+++ b/glance_store/_drivers/cinder.py
@@ -435,10 +435,15 @@ class Store(glance_store.driver.Store):
 
     def __init__(self, *args, **kargs):
         super(Store, self).__init__(*args, **kargs)
+        if self.backend_group:
+            self._set_url_prefix()
 
     def get_schemes(self):
         return ('cinder',)
 
+    def _set_url_prefix(self):
+        self._url_prefix = "cinder://"
+
     def _check_context(self, context, require_tenant=False):
         user_overriden = is_user_overriden(self.conf,
                                            backend=self.backend_group)
diff --git a/glance_store/_drivers/filesystem.py b/glance_store/_drivers/filesystem.py
index 4181f9d..f5e7871 100644
--- a/glance_store/_drivers/filesystem.py
+++ b/glance_store/_drivers/filesystem.py
@@ -460,10 +460,16 @@ class Store(glance_store.driver.Store):
                                         reverse=True)
 
         self._create_image_directories(directory_paths)
+        if self.backend_group:
+            self._set_url_prefix()
 
         if meta_file:
             self._validate_metadata(meta_file)
 
+    def _set_url_prefix(self):
+        path = self._find_best_datadir(0)
+        self._url_prefix = "%s://%s" % ('file', path)
+
     def _check_directory_paths(self, datadir_path, directory_paths,
                                priority_paths):
         """
diff --git a/glance_store/_drivers/http.py b/glance_store/_drivers/http.py
index d2a821a..b0b70f5 100644
--- a/glance_store/_drivers/http.py
+++ b/glance_store/_drivers/http.py
@@ -191,6 +191,16 @@ class Store(glance_store.driver.Store):
                      capabilities.BitMasks.DRIVER_REUSABLE)
     OPTIONS = _HTTP_OPTS
 
+    def __init__(self, *args, **kargs):
+        super(Store, self).__init__(*args, **kargs)
+        if self.backend_group:
+            self._set_url_prefix()
+
+    def _set_url_prefix(self):
+        # NOTE(abhishekk): HTTP store url either starts with http
+        # or https, so default _url_prefix is set to http.
+        self._url_prefix = "http"
+
     @capabilities.check
     def get(self, location, offset=0, chunk_size=None, context=None):
         """
diff --git a/glance_store/_drivers/rbd.py b/glance_store/_drivers/rbd.py
index 74a80fb..98cb284 100644
--- a/glance_store/_drivers/rbd.py
+++ b/glance_store/_drivers/rbd.py
@@ -319,6 +319,23 @@ class Store(driver.Store):
             LOG.error(reason)
             raise exceptions.BadStoreConfiguration(store_name='rbd',
                                                    reason=reason)
+        if self.backend_group:
+            self._set_url_prefix()
+
+    def _set_url_prefix(self):
+        fsid = None
+        with self.get_connection(conffile=self.conf_file,
+                                 rados_id=self.user) as conn:
+            if hasattr(conn, 'get_fsid'):
+                fsid = encodeutils.safe_decode(conn.get_fsid())
+
+        if fsid and self.pool:
+            # ensure nothing contains / or any other url-unsafe character
+            safe_fsid = urllib.parse.quote(fsid, '')
+            safe_pool = urllib.parse.quote(self.pool, '')
+            self._url_prefix = "rbd://%s/%s/" % (safe_fsid, safe_pool)
+        else:
+            self._url_prefix = "rbd://"
 
     @capabilities.check
     def get(self, location, offset=0, chunk_size=None, context=None):
diff --git a/glance_store/_drivers/sheepdog.py b/glance_store/_drivers/sheepdog.py
index 591efe8..2b1f451 100644
--- a/glance_store/_drivers/sheepdog.py
+++ b/glance_store/_drivers/sheepdog.py
@@ -270,6 +270,10 @@ class Store(glance_store.driver.Store):
     def get_schemes(self):
         return ('sheepdog',)
 
+    def _set_url_prefix(self):
+        self._url_prefix = "%s://%s:%s:" % (
+            'sheepdog', self.addr, self.port)
+
     def configure_add(self):
         """
         Configure the Store to use the stored configuration options
@@ -304,6 +308,9 @@ class Store(glance_store.driver.Store):
             raise exceptions.BadStoreConfiguration(store_name='sheepdog',
                                                    reason=reason)
 
+        if self.backend_group:
+            self._set_url_prefix()
+
     @capabilities.check
     def get(self, location, offset=0, chunk_size=None, context=None):
         """
diff --git a/glance_store/_drivers/swift/store.py b/glance_store/_drivers/swift/store.py
index 706d352..47c892d 100644
--- a/glance_store/_drivers/swift/store.py
+++ b/glance_store/_drivers/swift/store.py
@@ -1222,6 +1222,9 @@ class BaseStore(driver.Store):
         msg = _("There is no Connection Manager implemented for %s class.")
         raise NotImplementedError(msg % self.__class__.__name__)
 
+    def _set_url_prefix(self, context=None):
+        raise NotImplementedError()
+
 
 class SingleTenantStore(BaseStore):
     EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>"
@@ -1280,6 +1283,50 @@ class SingleTenantStore(BaseStore):
             LOG.error(reason)
             raise exceptions.BadStoreConfiguration(store_name="swift",
                                                    reason=reason)
+        if self.backend_group:
+            self._set_url_prefix()
+
+    def _get_credstring(self):
+        if self.user and self.key:
+            return '%s:%s' % (urllib.parse.quote(self.user),
+                              urllib.parse.quote(self.key))
+        return ''
+
+    def _set_url_prefix(self, context=None):
+        auth_or_store_url = self.auth_address
+        if auth_or_store_url.startswith('http://'):
+            auth_or_store_url = auth_or_store_url[len('http://'):]
+        elif auth_or_store_url.startswith('https://'):
+            auth_or_store_url = auth_or_store_url[len('https://'):]
+
+        credstring = self._get_credstring()
+        auth_or_store_url = auth_or_store_url.strip('/')
+        container = self.container.strip('/')
+
+        if sutils.is_multiple_swift_store_accounts_enabled(
+                self.conf, backend=self.backend_group):
+            include_creds = False
+        else:
+            include_creds = True
+
+        if not include_creds:
+            store = getattr(self.conf,
+                            self.backend_group).default_swift_reference
+
+            self._url_prefix = '%s://%s/%s/' % (
+                'swift+config', store, container)
+            return
+
+        if self.scheme == 'swift+config':
+            if self.ssl_enabled:
+                self.scheme = 'swift+https'
+            else:
+                self.scheme = 'swift+http'
+        if credstring != '':
+            credstring = "%s@" % credstring
+
+        self._url_prefix = '%s://%s%s/%s/' % (
+            self.scheme, credstring, auth_or_store_url, container)
 
     def create_location(self, image_id, context=None):
         container_name = self.get_container_name(image_id, self.container)
@@ -1502,6 +1549,11 @@ class MultiTenantStore(BaseStore):
         return StoreLocation(specs, self.conf,
                              backend_group=self.backend_group)
 
+    def _set_url_prefix(self, context=None):
+        ep = self._get_endpoint(context)
+        self._url_prefix = "%s://%s:%s_" % (
+            self.scheme, ep, self.container)
+
     def get_connection(self, location, context=None):
         return swiftclient.Connection(
             preauthurl=location.swift_url,
@@ -1536,6 +1588,9 @@ class MultiTenantStore(BaseStore):
         project_domain_name = default_swift_reference.get(
             'project_domain_name')
 
+        if self.backend_group:
+            self._set_url_prefix(context=context)
+
         # create client for multitenant user(trustor)
         trustor_auth = ks_identity.V3Token(auth_url=auth_address,
                                            token=context.auth_token,
diff --git a/glance_store/_drivers/vmware_datastore.py b/glance_store/_drivers/vmware_datastore.py
index 76a9edd..9ca691e 100644
--- a/glance_store/_drivers/vmware_datastore.py
+++ b/glance_store/_drivers/vmware_datastore.py
@@ -515,6 +515,19 @@ class Store(glance_store.Store):
             store_conf = self.conf.glance_store
 
         self.store_image_dir = store_conf.vmware_store_image_dir
+        if self.backend_group:
+            self._set_url_prefix()
+
+    def _set_url_prefix(self):
+        path = os.path.join(DS_URL_PREFIX,
+                            self.store_image_dir)
+
+        if netutils.is_valid_ipv6(self.server_host):
+            self._url_prefix = '%s://[%s]%s' % (self.scheme,
+                                                self.server_host, path)
+        else:
+            self._url_prefix = '%s://%s%s' % (self.scheme,
+                                              self.server_host, path)
 
     def select_datastore(self, image_size):
         """Select a datastore with free space larger than image size."""
diff --git a/glance_store/driver.py b/glance_store/driver.py
index 0881a47..b11012a 100644
--- a/glance_store/driver.py
+++ b/glance_store/driver.py
@@ -61,6 +61,7 @@ class Store(capabilities.StoreCapability):
         self.conf = conf
         self.backend_group = backend
         self.store_location_class = None
+        self._url_prefix = None
 
         try:
             if self.OPTIONS is not None:
@@ -75,6 +76,10 @@ class Store(capabilities.StoreCapability):
         except cfg.DuplicateOptError:
             pass
 
+    @property
+    def url_prefix(self):
+        return self._url_prefix
+
     def configure(self, re_raise_bsc=False):
         """
         Configure the store to use the stored configuration options
diff --git a/glance_store/tests/unit/test_multistore_cinder.py b/glance_store/tests/unit/test_multistore_cinder.py
index 57c89d3..d3ebd62 100644
--- a/glance_store/tests/unit/test_multistore_cinder.py
+++ b/glance_store/tests/unit/test_multistore_cinder.py
@@ -67,8 +67,8 @@ class TestMultiCinderStore(base.MultiStoreBaseTest,
 
         # Ensure stores + locations cleared
         location.SCHEME_TO_CLS_BACKEND_MAP = {}
-
         store.create_multi_stores(self.conf)
+
         self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP',
                         dict())
         self.test_dir = self.useFixture(fixtures.TempDir()).path
@@ -89,6 +89,9 @@ class TestMultiCinderStore(base.MultiStoreBaseTest,
                                   auth_token='fake_token',
                                   tenant='fake_tenant')
 
+    def test_location_url_prefix_is_set(self):
+        self.assertEqual("cinder://", self.store.url_prefix)
+
     def test_get_cinderclient(self):
         cc = cinder.get_cinderclient(self.conf, self.context,
                                      backend="cinder1")
diff --git a/glance_store/tests/unit/test_multistore_filesystem.py b/glance_store/tests/unit/test_multistore_filesystem.py
index 62d68e7..b4e2b87 100644
--- a/glance_store/tests/unit/test_multistore_filesystem.py
+++ b/glance_store/tests/unit/test_multistore_filesystem.py
@@ -50,21 +50,21 @@ class TestMultiStore(base.MultiStoreBaseTest,
     def setUp(self):
         """Establish a clean test environment."""
         super(TestMultiStore, self).setUp()
-        enabled_backends = {
+        self.enabled_backends = {
             "file1": "file",
             "file2": "file",
         }
         self.conf = self._CONF
         self.conf(args=[])
         self.conf.register_opt(cfg.DictOpt('enabled_backends'))
-        self.config(enabled_backends=enabled_backends)
+        self.config(enabled_backends=self.enabled_backends)
         store.register_store_opts(self.conf)
         self.config(default_backend='file1', group='glance_store')
 
         # Ensure stores + locations cleared
         location.SCHEME_TO_CLS_BACKEND_MAP = {}
-
         store.create_multi_stores(self.conf)
+
         self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP',
                         dict())
         self.test_dir = self.useFixture(fixtures.TempDir()).path
@@ -96,6 +96,10 @@ class TestMultiStore(base.MultiStoreBaseTest,
         return self.store.add(expected_image_id, image_file,
                               expected_file_size)
 
+    def test_location_url_prefix_is_set(self):
+        expected_url_prefix = "file://%s" % self.test_dir
+        self.assertEqual(expected_url_prefix, self.store.url_prefix)
+
     def test_get(self):
         """Test a "normal" retrieval of an image in chunks."""
         # First add an image...
diff --git a/glance_store/tests/unit/test_multistore_rbd.py b/glance_store/tests/unit/test_multistore_rbd.py
index 363ed9f..75dd9bc 100644
--- a/glance_store/tests/unit/test_multistore_rbd.py
+++ b/glance_store/tests/unit/test_multistore_rbd.py
@@ -190,7 +190,9 @@ class TestMultiStore(base.MultiStoreBaseTest,
         # Ensure stores + locations cleared
         g_location.SCHEME_TO_CLS_BACKEND_MAP = {}
 
-        store.create_multi_stores(self.conf)
+        with mock.patch.object(rbd_store.Store, '_set_url_prefix'):
+            store.create_multi_stores(self.conf)
+
         self.addCleanup(setattr, g_location, 'SCHEME_TO_CLS_BACKEND_MAP',
                         dict())
         self.addCleanup(self.conf.reset)
@@ -212,6 +214,10 @@ class TestMultiStore(base.MultiStoreBaseTest,
         self.data_len = 3 * units.Ki
         self.data_iter = six.BytesIO(b'*' * self.data_len)
 
+    def test_location_url_prefix_is_set(self):
+        expected_url_prefix = "rbd://"
+        self.assertEqual(expected_url_prefix, self.store.url_prefix)
+
     def test_add_w_image_size_zero(self):
         """Assert that correct size is returned even though 0 was provided."""
         self.store.chunk_size = units.Ki
diff --git a/glance_store/tests/unit/test_multistore_sheepdog.py b/glance_store/tests/unit/test_multistore_sheepdog.py
index 999eba1..5d2cbc8 100644
--- a/glance_store/tests/unit/test_multistore_sheepdog.py
+++ b/glance_store/tests/unit/test_multistore_sheepdog.py
@@ -59,8 +59,8 @@ class TestSheepdogMultiStore(base.MultiStoreBaseTest,
 
         # Ensure stores + locations cleared
         location.SCHEME_TO_CLS_BACKEND_MAP = {}
-
         store.create_multi_stores(self.conf)
+
         self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP',
                         dict())
         self.addCleanup(self.conf.reset)
@@ -71,6 +71,10 @@ class TestSheepdogMultiStore(base.MultiStoreBaseTest,
                             'addr': '127.0.0.1',
                             'port': 7000}
 
+    def test_location_url_prefix_is_set(self):
+        expected_url_prefix = "sheepdog://127.0.0.1:7000:"
+        self.assertEqual(expected_url_prefix, self.store.url_prefix)
+
     @mock.patch.object(sheepdog.SheepdogImage, 'write')
     @mock.patch.object(sheepdog.SheepdogImage, 'create')
     @mock.patch.object(sheepdog.SheepdogImage, 'exist')
diff --git a/glance_store/tests/unit/test_multistore_vmware.py b/glance_store/tests/unit/test_multistore_vmware.py
index e88b53a..c4ab983 100644
--- a/glance_store/tests/unit/test_multistore_vmware.py
+++ b/glance_store/tests/unit/test_multistore_vmware.py
@@ -115,8 +115,8 @@ class TestMultiStore(base.MultiStoreBaseTest,
                     vmware_store_image_dir='/openstack_glance_1')
         # Ensure stores + locations cleared
         location.SCHEME_TO_CLS_BACKEND_MAP = {}
-
         store.create_multi_stores(self.conf)
+
         self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP',
                         dict())
         self.addCleanup(self.conf.reset)
@@ -131,6 +131,10 @@ class TestMultiStore(base.MultiStoreBaseTest,
     def _mock_http_connection(self):
         return mock.patch('six.moves.http_client.HTTPConnection')
 
+    def test_location_url_prefix_is_set(self):
+        expected_url_prefix = "vsphere://127.0.0.1/openstack_glance"
+        self.assertEqual(expected_url_prefix, self.store.url_prefix)
+
     @mock.patch('oslo_vmware.api.VMwareAPISession')
     def test_get(self, mock_api_session):
         """Test a "normal" retrieval of an image in chunks."""
diff --git a/glance_store/tests/unit/test_swift_store_multibackend.py b/glance_store/tests/unit/test_swift_store_multibackend.py
index 07c44d9..b64538a 100644
--- a/glance_store/tests/unit/test_swift_store_multibackend.py
+++ b/glance_store/tests/unit/test_swift_store_multibackend.py
@@ -964,6 +964,12 @@ class SwiftTests(object):
         self.assertEqual(expected_swift_contents, new_image_contents)
         self.assertEqual(expected_swift_size, new_image_swift_size)
 
+    def test_location_url_prefix_is_set(self):
+        self.store = Store(self.conf, backend="swift1")
+        self.store.configure()
+        expected_url_prefix = "swift+config://ref1/glance/"
+        self.assertEqual(expected_url_prefix, self.store.url_prefix)
+
     def test_add_already_existing(self):
         """
         Tests that adding an image with an existing identifier
@@ -1245,17 +1251,19 @@ class SwiftTests(object):
 
     def test_init_client_multi_tenant(self):
         """Test that keystone client was initialized correctly"""
-        self._init_client(verify=True, swift_store_multi_tenant=True,
-                          swift_store_config_file=None)
+        with mock.patch.object(swift.MultiTenantStore, '_set_url_prefix'):
+            self._init_client(verify=True, swift_store_multi_tenant=True,
+                              swift_store_config_file=None)
 
     def test_init_client_multi_tenant_insecure(self):
         """
         Test that keystone client was initialized correctly with no
         certificate verification.
         """
-        self._init_client(verify=False, swift_store_multi_tenant=True,
-                          swift_store_auth_insecure=True,
-                          swift_store_config_file=None)
+        with mock.patch.object(swift.MultiTenantStore, '_set_url_prefix'):
+            self._init_client(verify=False, swift_store_multi_tenant=True,
+                              swift_store_auth_insecure=True,
+                              swift_store_config_file=None)
 
     @mock.patch("glance_store._drivers.swift.store.ks_identity")
     @mock.patch("glance_store._drivers.swift.store.ks_session")