diff options
| author | Corey Bryant <corey.bryant@canonical.com> | 2020-10-22 10:55:33 -0400 |
|---|---|---|
| committer | Corey Bryant <corey.bryant@canonical.com> | 2021-02-12 19:36:14 +0000 |
| commit | 0a3851af306095ea89f6006fb0caa155e8013597 (patch) | |
| tree | 149e1b14a348d37879de0f8ff57fd3de5c098a84 | |
| parent | 1b24dd6f45372a180f80a7b13dbddbad4299a073 (diff) | |
| download | cinder-0a3851af306095ea89f6006fb0caa155e8013597.tar.gz | |
RBD: Retry delete if VolumeIsBusy in _copy_image_to_volume
Cinder can fail to create an image-based volume if RBD mirroring
is enabled. With the journaling-based approach to RBD mirroring,
ceph will still create a snapshot as a result of volume creation.
The volume create in _create_from_image_download() results in
a snapshot getting created, resulting in a race where delete_volume()
gets a VolumeIsBusy exception.
Change-Id: Ib80e04512ec34a390e9e17af2f3544e18cad8598
Closes-Bug: #1900775
(cherry picked from commit 6231d26667508e34eb0a985f1f7ced1dbe623e1a)
(cherry picked from commit e1ed30838c3d72e70e3e06d5e7bc6f18d6bc4aad)
(cherry picked from commit 4f40f059f81ac27c92e4238f78ba649b69b09dd0)
| -rw-r--r-- | cinder/tests/unit/volume/drivers/test_rbd.py | 23 | ||||
| -rw-r--r-- | cinder/volume/drivers/rbd.py | 8 |
2 files changed, 27 insertions, 4 deletions
diff --git a/cinder/tests/unit/volume/drivers/test_rbd.py b/cinder/tests/unit/volume/drivers/test_rbd.py index e1ba497b2..b918c3b93 100644 --- a/cinder/tests/unit/volume/drivers/test_rbd.py +++ b/cinder/tests/unit/volume/drivers/test_rbd.py @@ -1308,17 +1308,29 @@ class RBDTestCase(test.TestCase): self.driver._is_cloneable(location, {'disk_format': f})) self.assertTrue(mock_get_fsid.called) - def _copy_image(self): + def _copy_image(self, volume_busy=False): with mock.patch.object(tempfile, 'NamedTemporaryFile'): with mock.patch.object(os.path, 'exists') as mock_exists: mock_exists.return_value = True with mock.patch.object(image_utils, 'fetch_to_raw'): - with mock.patch.object(self.driver, 'delete_volume'): + with mock.patch.object(self.driver, 'delete_volume') \ + as mock_dv: with mock.patch.object(self.driver, '_resize'): mock_image_service = mock.MagicMock() args = [None, self.volume_a, mock_image_service, None] - self.driver.copy_image_to_volume(*args) + if volume_busy: + mock_dv.side_effect = ( + exception.VolumeIsBusy("doh")) + self.assertRaises( + exception.VolumeIsBusy, + self.driver.copy_image_to_volume, + *args) + self.assertEqual( + self.cfg.rados_connection_retries, + mock_dv.call_count) + else: + self.driver.copy_image_to_volume(*args) @mock.patch('cinder.volume.drivers.rbd.fileutils.delete_if_exists') @mock.patch('cinder.volume.volume_utils.check_encryption_provider', @@ -1368,6 +1380,11 @@ class RBDTestCase(test.TestCase): self.cfg.image_conversion_dir = '/var/run/cinder/tmp' self._copy_image_encrypted() + @common_mocks + def test_copy_image_busy_volume(self): + self.cfg.image_conversion_dir = '/var/run/cinder/tmp' + self._copy_image(volume_busy=True) + @ddt.data(True, False) @common_mocks @mock.patch('cinder.volume.drivers.rbd.RBDDriver._get_usage_info') diff --git a/cinder/volume/drivers/rbd.py b/cinder/volume/drivers/rbd.py index a3a48607e..ccb72f92b 100644 --- a/cinder/volume/drivers/rbd.py +++ b/cinder/volume/drivers/rbd.py @@ -1571,7 +1571,13 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, if encrypted: self._encrypt_image(context, volume, tmp_dir, tmp.name) - self.delete_volume(volume) + @utils.retry(exception.VolumeIsBusy, + self.configuration.rados_connection_interval, + self.configuration.rados_connection_retries) + def _delete_volume(volume): + self.delete_volume(volume) + + _delete_volume(volume) chunk_size = self.configuration.rbd_store_chunk_size * units.Mi order = int(math.log(chunk_size, 2)) |