diff options
Diffstat (limited to 'nova')
| -rw-r--r-- | nova/conf/glance.py | 59 | ||||
| -rw-r--r-- | nova/image/glance.py | 67 | ||||
| -rw-r--r-- | nova/storage/rbd_utils.py | 31 | ||||
| -rw-r--r-- | nova/tests/unit/image/test_glance.py | 135 | ||||
| -rw-r--r-- | nova/tests/unit/storage/test_rbd.py | 40 |
5 files changed, 319 insertions, 13 deletions
diff --git a/nova/conf/glance.py b/nova/conf/glance.py index 3391621c53..8838160411 100644 --- a/nova/conf/glance.py +++ b/nova/conf/glance.py @@ -148,9 +148,64 @@ Related options: * The value of this option may be used if both verify_glance_signatures and enable_certificate_validation are enabled. """), + cfg.BoolOpt('enable_rbd_download', + default=False, + help=""" +Enable download of Glance images directly via RBD. + +Allow compute hosts to quickly download and cache images localy directly +from Ceph rather than slow dowloads from the Glance API. This can +reduce download time for images in the ten to hundreds of GBs from tens of +minutes to tens of seconds, but requires a Ceph-based deployment and access +from the compute nodes to Ceph. + +Related options: + +* ``[glance] rbd_user`` +* ``[glance] rbd_connect_timeout`` +* ``[glance] rbd_pool`` +* ``[glance] rbd_ceph_conf`` +"""), + cfg.StrOpt('rbd_user', + default='', + help=""" +The RADOS client name for accessing Glance images stored as rbd volumes. + +Related options: + +* This option is only used if ``[glance] enable_rbd_download`` is set to True. +"""), + cfg.IntOpt('rbd_connect_timeout', + default=5, + help=""" +The RADOS client timeout in seconds when initially connecting to the cluster. + +Related options: + +* This option is only used if ``[glance] enable_rbd_download`` is set to True. +"""), + cfg.StrOpt('rbd_pool', + default='', + help=""" +The RADOS pool in which the Glance images are stored as rbd volumes. + +Related options: + +* This option is only used if ``[glance] enable_rbd_download`` is set to True. +"""), + cfg.StrOpt('rbd_ceph_conf', + default='', + help=""" +Path to the ceph configuration file to use. + +Related options: + +* This option is only used if ``[glance] enable_rbd_download`` is set to True. +"""), + cfg.BoolOpt('debug', - default=False, - help='Enable or disable debug logging with glanceclient.') + default=False, + help='Enable or disable debug logging with glanceclient.') ] deprecated_ksa_opts = { diff --git a/nova/image/glance.py b/nova/image/glance.py index be0c1eeccd..5e72679a2a 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -24,12 +24,14 @@ import re import stat import sys import time +import urllib.parse as urlparse import cryptography from cursive import certificate_utils from cursive import exception as cursive_exception from cursive import signature_utils import glanceclient +from glanceclient.common import utils as glance_utils import glanceclient.exc from glanceclient.v2 import schemas from keystoneauth1 import loading as ks_loading @@ -39,7 +41,6 @@ from oslo_utils import excutils from oslo_utils import timeutils import six from six.moves import range -import six.moves.urllib.parse as urlparse import nova.conf from nova import exception @@ -221,6 +222,51 @@ class GlanceImageServiceV2(object): # to be added here. self._download_handlers = {} + if CONF.glance.enable_rbd_download: + self._download_handlers['rbd'] = self.rbd_download + + def rbd_download(self, context, url_parts, dst_path, metadata=None): + """Use an explicit rbd call to download an image. + + :param context: The `nova.context.RequestContext` object for the + request + :param url_parts: Parts of URL pointing to the image location + :param dst_path: Filepath to transfer the image file to. + :param metadata: Image location metadata (currently unused) + """ + + # avoid circular import + from nova.storage import rbd_utils + try: + # Parse the RBD URL from url_parts, it should consist of 4 + # sections and be in the format of: + # <cluster_uuid>/<pool_name>/<image_uuid>/<snapshot_name> + url_path = str(urlparse.unquote(url_parts.path)) + cluster_uuid, pool_name, image_uuid, snapshot_name = ( + url_path.split('/')) + except ValueError as e: + msg = f"Invalid RBD URL format: {e}" + LOG.error(msg) + raise nova.exception.InvalidParameterValue(msg) + + rbd_driver = rbd_utils.RBDDriver( + user=CONF.glance.rbd_user, + pool=CONF.glance.rbd_pool, + ceph_conf=CONF.glance.rbd_ceph_conf, + connect_timeout=CONF.glance.rbd_connect_timeout) + + try: + LOG.debug("Attempting to export RBD image: " + "[pool_name: %s] [image_uuid: %s] " + "[snapshot_name: %s] [dst_path: %s]", + pool_name, image_uuid, snapshot_name, dst_path) + + rbd_driver.export_image(dst_path, image_uuid, + snapshot_name, pool_name) + except Exception as e: + LOG.error("Error during RBD image export: %s", e) + raise nova.exception.CouldNotFetchImage(image_id=image_uuid) + def show(self, context, image_id, include_locations=False, show_deleted=True): """Returns a dict with image data for the given opaque image id. @@ -299,7 +345,13 @@ class GlanceImageServiceV2(object): def download(self, context, image_id, data=None, dst_path=None, trusted_certs=None): """Calls out to Glance for data and writes data.""" - if CONF.glance.allowed_direct_url_schemes and dst_path is not None: + + # First, check if image could be directly downloaded by special handler + # TODO(stephenfin): Remove check for 'allowed_direct_url_schemes' when + # we clean up tests since it's not used elsewhere + if ((CONF.glance.allowed_direct_url_schemes or + self._download_handlers) and dst_path is not None + ): image = self.show(context, image_id, include_locations=True) for entry in image.get('locations', []): loc_url = entry['url'] @@ -310,10 +362,21 @@ class GlanceImageServiceV2(object): try: xfer_method(context, o, dst_path, loc_meta) LOG.info("Successfully transferred using %s", o.scheme) + + # Load chunks from the downloaded image file + # for verification (if required) + with open(dst_path, 'rb') as fh: + downloaded_length = os.path.getsize(dst_path) + image_chunks = glance_utils.IterableWithLength(fh, + downloaded_length) + self._verify_and_write(context, image_id, + trusted_certs, image_chunks, None, None) return except Exception: LOG.exception("Download image error") + # By default (or if direct download has failed), use glance client call + # to fetch the image and fill image_chunks try: image_chunks = self._client.call( context, 2, 'data', args=(image_id,)) diff --git a/nova/storage/rbd_utils.py b/nova/storage/rbd_utils.py index bda1b5d542..22bafe5053 100644 --- a/nova/storage/rbd_utils.py +++ b/nova/storage/rbd_utils.py @@ -122,14 +122,16 @@ class RADOSClient(object): class RBDDriver(object): - def __init__(self): + def __init__(self, pool=None, user=None, ceph_conf=None, + connect_timeout=None): if rbd is None: raise RuntimeError(_('rbd python libraries not found')) - self.pool = CONF.libvirt.images_rbd_pool - self.rbd_user = CONF.libvirt.rbd_user - self.rbd_connect_timeout = CONF.libvirt.rbd_connect_timeout - self.ceph_conf = CONF.libvirt.images_rbd_ceph_conf + self.pool = pool or CONF.libvirt.images_rbd_pool + self.rbd_user = user or CONF.libvirt.rbd_user + self.rbd_connect_timeout = ( + connect_timeout or CONF.libvirt.rbd_connect_timeout) + self.ceph_conf = ceph_conf or CONF.libvirt.images_rbd_ceph_conf def _connect_to_rados(self, pool=None): client = rados.Rados(rados_id=self.rbd_user, @@ -335,6 +337,25 @@ class RBDDriver(object): args += self.ceph_args() processutils.execute('rbd', 'import', *args) + def export_image(self, base, name, snap, pool=None): + """Export RBD volume to image file. + + Uses the command line export to export rbd volume snapshot to + local image file. + + :base: Path to image file + :name: Name of RBD volume + :snap: Name of RBD snapshot + :pool: Name of RBD pool + """ + if pool is None: + pool = self.pool + + args = ['--pool', pool, '--image', name, '--path', base, + '--snap', snap] + args += self.ceph_args() + processutils.execute('rbd', 'export', *args) + def _destroy_volume(self, client, volume, pool=None): """Destroy an RBD volume, retrying as needed. """ diff --git a/nova/tests/unit/image/test_glance.py b/nova/tests/unit/image/test_glance.py index a672a967ed..170b2282f3 100644 --- a/nova/tests/unit/image/test_glance.py +++ b/nova/tests/unit/image/test_glance.py @@ -16,6 +16,7 @@ import copy import datetime +import urllib.parse as urlparse import cryptography from cursive import exception as cursive_exception @@ -36,6 +37,7 @@ from nova import exception from nova.image import glance from nova import objects from nova import service_auth +from nova.storage import rbd_utils from nova import test @@ -686,9 +688,14 @@ class TestDownloadNoDirectUri(test.NoDBTestCase): with testtools.ExpectedException(exception.ImageUnacceptable): service.download(ctx, mock.sentinel.image_id) + # TODO(stephenfin): Drop this test since it's not possible to run in + # production + @mock.patch('os.path.getsize', return_value=1) + @mock.patch.object(six.moves.builtins, 'open') @mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method') @mock.patch('nova.image.glance.GlanceImageServiceV2.show') - def test_download_direct_file_uri_v2(self, show_mock, get_tran_mock): + def test_download_direct_file_uri_v2( + self, show_mock, get_tran_mock, open_mock, getsize_mock): self.flags(allowed_direct_url_schemes=['file'], group='glance') show_mock.return_value = { 'locations': [ @@ -702,6 +709,8 @@ class TestDownloadNoDirectUri(test.NoDBTestCase): get_tran_mock.return_value = tran_mod client = mock.MagicMock() ctx = mock.sentinel.ctx + writer = mock.MagicMock() + open_mock.return_value = writer service = glance.GlanceImageServiceV2(client) res = service.download(ctx, mock.sentinel.image_id, dst_path=mock.sentinel.dst_path) @@ -716,6 +725,76 @@ class TestDownloadNoDirectUri(test.NoDBTestCase): mock.sentinel.dst_path, mock.sentinel.loc_meta) + @mock.patch('glanceclient.common.utils.IterableWithLength') + @mock.patch('os.path.getsize', return_value=1) + @mock.patch.object(six.moves.builtins, 'open') + @mock.patch('nova.image.glance.LOG') + @mock.patch('nova.image.glance.GlanceImageServiceV2._get_verifier') + @mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method') + @mock.patch('nova.image.glance.GlanceImageServiceV2.show') + def test_download_direct_rbd_uri_v2( + self, show_mock, get_tran_mock, get_verifier_mock, log_mock, + open_mock, getsize_mock, iterable_with_length_mock): + self.flags(enable_rbd_download=True, group='glance') + show_mock.return_value = { + 'locations': [ + { + 'url': 'rbd://cluster_uuid/pool_name/image_uuid/snapshot', + 'metadata': mock.sentinel.loc_meta + } + ] + } + tran_mod = mock.MagicMock() + get_tran_mock.return_value = tran_mod + client = mock.MagicMock() + ctx = mock.sentinel.ctx + writer = mock.MagicMock() + open_mock.return_value = writer + iterable_with_length_mock.return_value = ["rbd1", "rbd2"] + service = glance.GlanceImageServiceV2(client) + + verifier = mock.MagicMock() + get_verifier_mock.return_value = verifier + + res = service.download(ctx, mock.sentinel.image_id, + dst_path=mock.sentinel.dst_path, + trusted_certs=mock.sentinel.trusted_certs) + + self.assertIsNone(res) + show_mock.assert_called_once_with(ctx, + mock.sentinel.image_id, + include_locations=True) + tran_mod.assert_called_once_with(ctx, mock.ANY, + mock.sentinel.dst_path, + mock.sentinel.loc_meta) + open_mock.assert_called_once_with(mock.sentinel.dst_path, 'rb') + get_tran_mock.assert_called_once_with('rbd') + + # no client call, chunks were read right after xfer_mod.download: + client.call.assert_not_called() + + # verifier called with the value we got from rbd download + verifier.update.assert_has_calls( + [ + mock.call("rbd1"), + mock.call("rbd2") + ] + ) + verifier.verify.assert_called() + log_mock.info.assert_has_calls( + [ + mock.call('Successfully transferred using %s', 'rbd'), + mock.call( + 'Image signature verification succeeded for image %s', + mock.sentinel.image_id) + ] + ) + + # not opened for writing (already written) + self.assertFalse(open_mock(mock.sentinel.dst_path, 'rw').called) + # write not called (written by rbd download) + writer.write.assert_not_called() + @mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method') @mock.patch('nova.image.glance.GlanceImageServiceV2.show') @mock.patch('nova.image.glance.GlanceImageServiceV2._safe_fsync') @@ -1249,6 +1328,60 @@ class TestIsImageAvailable(test.NoDBTestCase): self.assertTrue(res) +class TestRBDDownload(test.NoDBTestCase): + + def setUp(self): + super(TestRBDDownload, self).setUp() + loc_url = "rbd://ce2d1ace/images/b86d6d06-faac/snap" + self.url_parts = urlparse.urlparse(loc_url) + self.image_uuid = "b86d6d06-faac" + self.pool_name = "images" + self.snapshot_name = "snap" + + @mock.patch.object(rbd_utils.RBDDriver, 'export_image') + @mock.patch.object(rbd_utils, 'rbd') + def test_rbd_download_success(self, mock_rbd, mock_export_image): + client = mock.MagicMock() + ctx = mock.sentinel.ctx + service = glance.GlanceImageServiceV2(client) + + service.rbd_download(ctx, self.url_parts, mock.sentinel.dst_path) + + # Assert that we attempt to export using the correct rbd pool, volume + # and snapshot given the provided URL + mock_export_image.assert_called_once_with(mock.sentinel.dst_path, + self.image_uuid, + self.snapshot_name, + self.pool_name) + + def test_rbd_download_broken_url(self): + client = mock.MagicMock() + ctx = mock.sentinel.ctx + service = glance.GlanceImageServiceV2(client) + + wrong_url = "http://www.example.com" + wrong_url_parts = urlparse.urlparse(wrong_url) + + # Assert InvalidParameterValue is raised when we can't parse the URL + self.assertRaises( + exception.InvalidParameterValue, service.rbd_download, ctx, + wrong_url_parts, mock.sentinel.dst_path) + + @mock.patch('nova.storage.rbd_utils.RBDDriver.export_image') + @mock.patch.object(rbd_utils, 'rbd') + def test_rbd_download_export_failure(self, mock_rbd, mock_export_image): + client = mock.MagicMock() + ctx = mock.sentinel.ctx + service = glance.GlanceImageServiceV2(client) + + mock_export_image.side_effect = Exception + + # Assert CouldNotFetchImage is raised when the export fails + self.assertRaisesRegex( + exception.CouldNotFetchImage, self.image_uuid, + service.rbd_download, ctx, self.url_parts, mock.sentinel.dst_path) + + class TestShow(test.NoDBTestCase): """Tests the show method of the GlanceImageServiceV2.""" diff --git a/nova/tests/unit/storage/test_rbd.py b/nova/tests/unit/storage/test_rbd.py index fd9372b870..5a39bdbd5a 100644 --- a/nova/tests/unit/storage/test_rbd.py +++ b/nova/tests/unit/storage/test_rbd.py @@ -109,9 +109,11 @@ class RbdTestCase(test.NoDBTestCase): self.rbd_pool = 'rbd' self.rbd_connect_timeout = 5 - self.flags(images_rbd_pool=self.rbd_pool, group='libvirt') - self.flags(rbd_connect_timeout=self.rbd_connect_timeout, - group='libvirt') + self.flags( + images_rbd_pool=self.images_rbd_pool, + images_rbd_ceph_conf='/foo/bar.conf', + rbd_connect_timeout=self.rbd_connect_timeout, + rbd_user='foo', group='libvirt') rados_patcher = mock.patch.object(rbd_utils, 'rados') self.mock_rados = rados_patcher.start() @@ -657,3 +659,35 @@ class RbdTestCase(test.NoDBTestCase): ceph_df_not_found = CEPH_DF.replace('rbd', 'vms') mock_execute.return_value = (ceph_df_not_found, '') self.assertRaises(exception.NotFound, self.driver.get_pool_info) + + @mock.patch('oslo_concurrency.processutils.execute') + def test_export_image(self, mock_execute): + self.driver.rbd_user = 'foo' + self.driver.export_image(mock.sentinel.dst_path, + mock.sentinel.name, + mock.sentinel.snap, + mock.sentinel.pool) + + mock_execute.assert_called_once_with( + 'rbd', 'export', + '--pool', mock.sentinel.pool, + '--image', mock.sentinel.name, + '--path', mock.sentinel.dst_path, + '--snap', mock.sentinel.snap, + '--id', 'foo', + '--conf', '/foo/bar.conf') + + @mock.patch('oslo_concurrency.processutils.execute') + def test_export_image_default_pool(self, mock_execute): + self.driver.export_image(mock.sentinel.dst_path, + mock.sentinel.name, + mock.sentinel.snap) + + mock_execute.assert_called_once_with( + 'rbd', 'export', + '--pool', self.rbd_pool, + '--image', mock.sentinel.name, + '--path', mock.sentinel.dst_path, + '--snap', mock.sentinel.snap, + '--id', 'foo', + '--conf', '/foo/bar.conf') |