summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCurt Moore <curt.moore@garmin.com>2018-06-11 10:08:57 -0500
committerJiří Suchomel <jiri.suchomel@suse.com>2020-08-31 15:14:11 +0200
commit61aeb1adbced8f0530f5d57bf7a6fe79c5f218d4 (patch)
tree1ffc1f1e26ad21bf22d579c8db045d17d9fca930
parentb5d48043466b53fbdfe7b93c2e4efd449904e593 (diff)
downloadnova-61aeb1adbced8f0530f5d57bf7a6fe79c5f218d4.tar.gz
Add ability to download Glance images into the libvirt image cache via RBD
This change allows compute hosts to quickly download and cache images on the local compute host directly from Ceph rather than slow dowloads from the Glance API. New '[glance]/enable_rbd_download' option is introduced to enable this behavior. This is slight change compared to the original idea described in the relevant blueprint where it was discussed to use (now obsolete) '[glance]/allowed_direct_url_schemes' option. Additionally, when an image signature verification is requested, it should be done also for the image fetched by the new download handler. This was completely missing so far. New '[glance]/rbd_{user,pool,ceph_conf,connect_timeout}' configurables are introduced to allow operators to configure access to the cluster hosting Glance without the need to use the existing '[libvirt]' specific configurables. nova.storage.rbd_utils.RBDDriver has also been modified to accept these but continues to default to the '[libvirt]' specific configurables for now. Change-Id: I3032bbe6bd2d6acc9ba0f0cac4d00ed4b4464ceb Implements: blueprint nova-image-download-via-rbd
-rw-r--r--nova/conf/glance.py59
-rw-r--r--nova/image/glance.py67
-rw-r--r--nova/storage/rbd_utils.py31
-rw-r--r--nova/tests/unit/image/test_glance.py135
-rw-r--r--nova/tests/unit/storage/test_rbd.py40
5 files changed, 319 insertions, 13 deletions
diff --git a/nova/conf/glance.py b/nova/conf/glance.py
index 3391621c53..8838160411 100644
--- a/nova/conf/glance.py
+++ b/nova/conf/glance.py
@@ -148,9 +148,64 @@ Related options:
* The value of this option may be used if both verify_glance_signatures and
enable_certificate_validation are enabled.
"""),
+ cfg.BoolOpt('enable_rbd_download',
+ default=False,
+ help="""
+Enable download of Glance images directly via RBD.
+
+Allow compute hosts to quickly download and cache images localy directly
+from Ceph rather than slow dowloads from the Glance API. This can
+reduce download time for images in the ten to hundreds of GBs from tens of
+minutes to tens of seconds, but requires a Ceph-based deployment and access
+from the compute nodes to Ceph.
+
+Related options:
+
+* ``[glance] rbd_user``
+* ``[glance] rbd_connect_timeout``
+* ``[glance] rbd_pool``
+* ``[glance] rbd_ceph_conf``
+"""),
+ cfg.StrOpt('rbd_user',
+ default='',
+ help="""
+The RADOS client name for accessing Glance images stored as rbd volumes.
+
+Related options:
+
+* This option is only used if ``[glance] enable_rbd_download`` is set to True.
+"""),
+ cfg.IntOpt('rbd_connect_timeout',
+ default=5,
+ help="""
+The RADOS client timeout in seconds when initially connecting to the cluster.
+
+Related options:
+
+* This option is only used if ``[glance] enable_rbd_download`` is set to True.
+"""),
+ cfg.StrOpt('rbd_pool',
+ default='',
+ help="""
+The RADOS pool in which the Glance images are stored as rbd volumes.
+
+Related options:
+
+* This option is only used if ``[glance] enable_rbd_download`` is set to True.
+"""),
+ cfg.StrOpt('rbd_ceph_conf',
+ default='',
+ help="""
+Path to the ceph configuration file to use.
+
+Related options:
+
+* This option is only used if ``[glance] enable_rbd_download`` is set to True.
+"""),
+
cfg.BoolOpt('debug',
- default=False,
- help='Enable or disable debug logging with glanceclient.')
+ default=False,
+ help='Enable or disable debug logging with glanceclient.')
]
deprecated_ksa_opts = {
diff --git a/nova/image/glance.py b/nova/image/glance.py
index be0c1eeccd..5e72679a2a 100644
--- a/nova/image/glance.py
+++ b/nova/image/glance.py
@@ -24,12 +24,14 @@ import re
import stat
import sys
import time
+import urllib.parse as urlparse
import cryptography
from cursive import certificate_utils
from cursive import exception as cursive_exception
from cursive import signature_utils
import glanceclient
+from glanceclient.common import utils as glance_utils
import glanceclient.exc
from glanceclient.v2 import schemas
from keystoneauth1 import loading as ks_loading
@@ -39,7 +41,6 @@ from oslo_utils import excutils
from oslo_utils import timeutils
import six
from six.moves import range
-import six.moves.urllib.parse as urlparse
import nova.conf
from nova import exception
@@ -221,6 +222,51 @@ class GlanceImageServiceV2(object):
# to be added here.
self._download_handlers = {}
+ if CONF.glance.enable_rbd_download:
+ self._download_handlers['rbd'] = self.rbd_download
+
+ def rbd_download(self, context, url_parts, dst_path, metadata=None):
+ """Use an explicit rbd call to download an image.
+
+ :param context: The `nova.context.RequestContext` object for the
+ request
+ :param url_parts: Parts of URL pointing to the image location
+ :param dst_path: Filepath to transfer the image file to.
+ :param metadata: Image location metadata (currently unused)
+ """
+
+ # avoid circular import
+ from nova.storage import rbd_utils
+ try:
+ # Parse the RBD URL from url_parts, it should consist of 4
+ # sections and be in the format of:
+ # <cluster_uuid>/<pool_name>/<image_uuid>/<snapshot_name>
+ url_path = str(urlparse.unquote(url_parts.path))
+ cluster_uuid, pool_name, image_uuid, snapshot_name = (
+ url_path.split('/'))
+ except ValueError as e:
+ msg = f"Invalid RBD URL format: {e}"
+ LOG.error(msg)
+ raise nova.exception.InvalidParameterValue(msg)
+
+ rbd_driver = rbd_utils.RBDDriver(
+ user=CONF.glance.rbd_user,
+ pool=CONF.glance.rbd_pool,
+ ceph_conf=CONF.glance.rbd_ceph_conf,
+ connect_timeout=CONF.glance.rbd_connect_timeout)
+
+ try:
+ LOG.debug("Attempting to export RBD image: "
+ "[pool_name: %s] [image_uuid: %s] "
+ "[snapshot_name: %s] [dst_path: %s]",
+ pool_name, image_uuid, snapshot_name, dst_path)
+
+ rbd_driver.export_image(dst_path, image_uuid,
+ snapshot_name, pool_name)
+ except Exception as e:
+ LOG.error("Error during RBD image export: %s", e)
+ raise nova.exception.CouldNotFetchImage(image_id=image_uuid)
+
def show(self, context, image_id, include_locations=False,
show_deleted=True):
"""Returns a dict with image data for the given opaque image id.
@@ -299,7 +345,13 @@ class GlanceImageServiceV2(object):
def download(self, context, image_id, data=None, dst_path=None,
trusted_certs=None):
"""Calls out to Glance for data and writes data."""
- if CONF.glance.allowed_direct_url_schemes and dst_path is not None:
+
+ # First, check if image could be directly downloaded by special handler
+ # TODO(stephenfin): Remove check for 'allowed_direct_url_schemes' when
+ # we clean up tests since it's not used elsewhere
+ if ((CONF.glance.allowed_direct_url_schemes or
+ self._download_handlers) and dst_path is not None
+ ):
image = self.show(context, image_id, include_locations=True)
for entry in image.get('locations', []):
loc_url = entry['url']
@@ -310,10 +362,21 @@ class GlanceImageServiceV2(object):
try:
xfer_method(context, o, dst_path, loc_meta)
LOG.info("Successfully transferred using %s", o.scheme)
+
+ # Load chunks from the downloaded image file
+ # for verification (if required)
+ with open(dst_path, 'rb') as fh:
+ downloaded_length = os.path.getsize(dst_path)
+ image_chunks = glance_utils.IterableWithLength(fh,
+ downloaded_length)
+ self._verify_and_write(context, image_id,
+ trusted_certs, image_chunks, None, None)
return
except Exception:
LOG.exception("Download image error")
+ # By default (or if direct download has failed), use glance client call
+ # to fetch the image and fill image_chunks
try:
image_chunks = self._client.call(
context, 2, 'data', args=(image_id,))
diff --git a/nova/storage/rbd_utils.py b/nova/storage/rbd_utils.py
index bda1b5d542..22bafe5053 100644
--- a/nova/storage/rbd_utils.py
+++ b/nova/storage/rbd_utils.py
@@ -122,14 +122,16 @@ class RADOSClient(object):
class RBDDriver(object):
- def __init__(self):
+ def __init__(self, pool=None, user=None, ceph_conf=None,
+ connect_timeout=None):
if rbd is None:
raise RuntimeError(_('rbd python libraries not found'))
- self.pool = CONF.libvirt.images_rbd_pool
- self.rbd_user = CONF.libvirt.rbd_user
- self.rbd_connect_timeout = CONF.libvirt.rbd_connect_timeout
- self.ceph_conf = CONF.libvirt.images_rbd_ceph_conf
+ self.pool = pool or CONF.libvirt.images_rbd_pool
+ self.rbd_user = user or CONF.libvirt.rbd_user
+ self.rbd_connect_timeout = (
+ connect_timeout or CONF.libvirt.rbd_connect_timeout)
+ self.ceph_conf = ceph_conf or CONF.libvirt.images_rbd_ceph_conf
def _connect_to_rados(self, pool=None):
client = rados.Rados(rados_id=self.rbd_user,
@@ -335,6 +337,25 @@ class RBDDriver(object):
args += self.ceph_args()
processutils.execute('rbd', 'import', *args)
+ def export_image(self, base, name, snap, pool=None):
+ """Export RBD volume to image file.
+
+ Uses the command line export to export rbd volume snapshot to
+ local image file.
+
+ :base: Path to image file
+ :name: Name of RBD volume
+ :snap: Name of RBD snapshot
+ :pool: Name of RBD pool
+ """
+ if pool is None:
+ pool = self.pool
+
+ args = ['--pool', pool, '--image', name, '--path', base,
+ '--snap', snap]
+ args += self.ceph_args()
+ processutils.execute('rbd', 'export', *args)
+
def _destroy_volume(self, client, volume, pool=None):
"""Destroy an RBD volume, retrying as needed.
"""
diff --git a/nova/tests/unit/image/test_glance.py b/nova/tests/unit/image/test_glance.py
index a672a967ed..170b2282f3 100644
--- a/nova/tests/unit/image/test_glance.py
+++ b/nova/tests/unit/image/test_glance.py
@@ -16,6 +16,7 @@
import copy
import datetime
+import urllib.parse as urlparse
import cryptography
from cursive import exception as cursive_exception
@@ -36,6 +37,7 @@ from nova import exception
from nova.image import glance
from nova import objects
from nova import service_auth
+from nova.storage import rbd_utils
from nova import test
@@ -686,9 +688,14 @@ class TestDownloadNoDirectUri(test.NoDBTestCase):
with testtools.ExpectedException(exception.ImageUnacceptable):
service.download(ctx, mock.sentinel.image_id)
+ # TODO(stephenfin): Drop this test since it's not possible to run in
+ # production
+ @mock.patch('os.path.getsize', return_value=1)
+ @mock.patch.object(six.moves.builtins, 'open')
@mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method')
@mock.patch('nova.image.glance.GlanceImageServiceV2.show')
- def test_download_direct_file_uri_v2(self, show_mock, get_tran_mock):
+ def test_download_direct_file_uri_v2(
+ self, show_mock, get_tran_mock, open_mock, getsize_mock):
self.flags(allowed_direct_url_schemes=['file'], group='glance')
show_mock.return_value = {
'locations': [
@@ -702,6 +709,8 @@ class TestDownloadNoDirectUri(test.NoDBTestCase):
get_tran_mock.return_value = tran_mod
client = mock.MagicMock()
ctx = mock.sentinel.ctx
+ writer = mock.MagicMock()
+ open_mock.return_value = writer
service = glance.GlanceImageServiceV2(client)
res = service.download(ctx, mock.sentinel.image_id,
dst_path=mock.sentinel.dst_path)
@@ -716,6 +725,76 @@ class TestDownloadNoDirectUri(test.NoDBTestCase):
mock.sentinel.dst_path,
mock.sentinel.loc_meta)
+ @mock.patch('glanceclient.common.utils.IterableWithLength')
+ @mock.patch('os.path.getsize', return_value=1)
+ @mock.patch.object(six.moves.builtins, 'open')
+ @mock.patch('nova.image.glance.LOG')
+ @mock.patch('nova.image.glance.GlanceImageServiceV2._get_verifier')
+ @mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method')
+ @mock.patch('nova.image.glance.GlanceImageServiceV2.show')
+ def test_download_direct_rbd_uri_v2(
+ self, show_mock, get_tran_mock, get_verifier_mock, log_mock,
+ open_mock, getsize_mock, iterable_with_length_mock):
+ self.flags(enable_rbd_download=True, group='glance')
+ show_mock.return_value = {
+ 'locations': [
+ {
+ 'url': 'rbd://cluster_uuid/pool_name/image_uuid/snapshot',
+ 'metadata': mock.sentinel.loc_meta
+ }
+ ]
+ }
+ tran_mod = mock.MagicMock()
+ get_tran_mock.return_value = tran_mod
+ client = mock.MagicMock()
+ ctx = mock.sentinel.ctx
+ writer = mock.MagicMock()
+ open_mock.return_value = writer
+ iterable_with_length_mock.return_value = ["rbd1", "rbd2"]
+ service = glance.GlanceImageServiceV2(client)
+
+ verifier = mock.MagicMock()
+ get_verifier_mock.return_value = verifier
+
+ res = service.download(ctx, mock.sentinel.image_id,
+ dst_path=mock.sentinel.dst_path,
+ trusted_certs=mock.sentinel.trusted_certs)
+
+ self.assertIsNone(res)
+ show_mock.assert_called_once_with(ctx,
+ mock.sentinel.image_id,
+ include_locations=True)
+ tran_mod.assert_called_once_with(ctx, mock.ANY,
+ mock.sentinel.dst_path,
+ mock.sentinel.loc_meta)
+ open_mock.assert_called_once_with(mock.sentinel.dst_path, 'rb')
+ get_tran_mock.assert_called_once_with('rbd')
+
+ # no client call, chunks were read right after xfer_mod.download:
+ client.call.assert_not_called()
+
+ # verifier called with the value we got from rbd download
+ verifier.update.assert_has_calls(
+ [
+ mock.call("rbd1"),
+ mock.call("rbd2")
+ ]
+ )
+ verifier.verify.assert_called()
+ log_mock.info.assert_has_calls(
+ [
+ mock.call('Successfully transferred using %s', 'rbd'),
+ mock.call(
+ 'Image signature verification succeeded for image %s',
+ mock.sentinel.image_id)
+ ]
+ )
+
+ # not opened for writing (already written)
+ self.assertFalse(open_mock(mock.sentinel.dst_path, 'rw').called)
+ # write not called (written by rbd download)
+ writer.write.assert_not_called()
+
@mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method')
@mock.patch('nova.image.glance.GlanceImageServiceV2.show')
@mock.patch('nova.image.glance.GlanceImageServiceV2._safe_fsync')
@@ -1249,6 +1328,60 @@ class TestIsImageAvailable(test.NoDBTestCase):
self.assertTrue(res)
+class TestRBDDownload(test.NoDBTestCase):
+
+ def setUp(self):
+ super(TestRBDDownload, self).setUp()
+ loc_url = "rbd://ce2d1ace/images/b86d6d06-faac/snap"
+ self.url_parts = urlparse.urlparse(loc_url)
+ self.image_uuid = "b86d6d06-faac"
+ self.pool_name = "images"
+ self.snapshot_name = "snap"
+
+ @mock.patch.object(rbd_utils.RBDDriver, 'export_image')
+ @mock.patch.object(rbd_utils, 'rbd')
+ def test_rbd_download_success(self, mock_rbd, mock_export_image):
+ client = mock.MagicMock()
+ ctx = mock.sentinel.ctx
+ service = glance.GlanceImageServiceV2(client)
+
+ service.rbd_download(ctx, self.url_parts, mock.sentinel.dst_path)
+
+ # Assert that we attempt to export using the correct rbd pool, volume
+ # and snapshot given the provided URL
+ mock_export_image.assert_called_once_with(mock.sentinel.dst_path,
+ self.image_uuid,
+ self.snapshot_name,
+ self.pool_name)
+
+ def test_rbd_download_broken_url(self):
+ client = mock.MagicMock()
+ ctx = mock.sentinel.ctx
+ service = glance.GlanceImageServiceV2(client)
+
+ wrong_url = "http://www.example.com"
+ wrong_url_parts = urlparse.urlparse(wrong_url)
+
+ # Assert InvalidParameterValue is raised when we can't parse the URL
+ self.assertRaises(
+ exception.InvalidParameterValue, service.rbd_download, ctx,
+ wrong_url_parts, mock.sentinel.dst_path)
+
+ @mock.patch('nova.storage.rbd_utils.RBDDriver.export_image')
+ @mock.patch.object(rbd_utils, 'rbd')
+ def test_rbd_download_export_failure(self, mock_rbd, mock_export_image):
+ client = mock.MagicMock()
+ ctx = mock.sentinel.ctx
+ service = glance.GlanceImageServiceV2(client)
+
+ mock_export_image.side_effect = Exception
+
+ # Assert CouldNotFetchImage is raised when the export fails
+ self.assertRaisesRegex(
+ exception.CouldNotFetchImage, self.image_uuid,
+ service.rbd_download, ctx, self.url_parts, mock.sentinel.dst_path)
+
+
class TestShow(test.NoDBTestCase):
"""Tests the show method of the GlanceImageServiceV2."""
diff --git a/nova/tests/unit/storage/test_rbd.py b/nova/tests/unit/storage/test_rbd.py
index fd9372b870..5a39bdbd5a 100644
--- a/nova/tests/unit/storage/test_rbd.py
+++ b/nova/tests/unit/storage/test_rbd.py
@@ -109,9 +109,11 @@ class RbdTestCase(test.NoDBTestCase):
self.rbd_pool = 'rbd'
self.rbd_connect_timeout = 5
- self.flags(images_rbd_pool=self.rbd_pool, group='libvirt')
- self.flags(rbd_connect_timeout=self.rbd_connect_timeout,
- group='libvirt')
+ self.flags(
+ images_rbd_pool=self.images_rbd_pool,
+ images_rbd_ceph_conf='/foo/bar.conf',
+ rbd_connect_timeout=self.rbd_connect_timeout,
+ rbd_user='foo', group='libvirt')
rados_patcher = mock.patch.object(rbd_utils, 'rados')
self.mock_rados = rados_patcher.start()
@@ -657,3 +659,35 @@ class RbdTestCase(test.NoDBTestCase):
ceph_df_not_found = CEPH_DF.replace('rbd', 'vms')
mock_execute.return_value = (ceph_df_not_found, '')
self.assertRaises(exception.NotFound, self.driver.get_pool_info)
+
+ @mock.patch('oslo_concurrency.processutils.execute')
+ def test_export_image(self, mock_execute):
+ self.driver.rbd_user = 'foo'
+ self.driver.export_image(mock.sentinel.dst_path,
+ mock.sentinel.name,
+ mock.sentinel.snap,
+ mock.sentinel.pool)
+
+ mock_execute.assert_called_once_with(
+ 'rbd', 'export',
+ '--pool', mock.sentinel.pool,
+ '--image', mock.sentinel.name,
+ '--path', mock.sentinel.dst_path,
+ '--snap', mock.sentinel.snap,
+ '--id', 'foo',
+ '--conf', '/foo/bar.conf')
+
+ @mock.patch('oslo_concurrency.processutils.execute')
+ def test_export_image_default_pool(self, mock_execute):
+ self.driver.export_image(mock.sentinel.dst_path,
+ mock.sentinel.name,
+ mock.sentinel.snap)
+
+ mock_execute.assert_called_once_with(
+ 'rbd', 'export',
+ '--pool', self.rbd_pool,
+ '--image', mock.sentinel.name,
+ '--path', mock.sentinel.dst_path,
+ '--snap', mock.sentinel.snap,
+ '--id', 'foo',
+ '--conf', '/foo/bar.conf')