diff options
Diffstat (limited to 'glanceclient')
| -rw-r--r-- | glanceclient/common/http.py | 16 | ||||
| -rw-r--r-- | glanceclient/common/utils.py | 28 | ||||
| -rw-r--r-- | glanceclient/exc.py | 2 | ||||
| -rw-r--r-- | glanceclient/tests/unit/test_http.py | 8 | ||||
| -rw-r--r-- | glanceclient/tests/unit/test_shell.py | 4 | ||||
| -rw-r--r-- | glanceclient/tests/unit/v1/test_shell.py | 8 | ||||
| -rw-r--r-- | glanceclient/tests/unit/v2/fixtures.py | 7 | ||||
| -rw-r--r-- | glanceclient/tests/unit/v2/test_images.py | 303 | ||||
| -rw-r--r-- | glanceclient/tests/unit/v2/test_shell_v2.py | 477 | ||||
| -rw-r--r-- | glanceclient/v1/shell.py | 10 | ||||
| -rw-r--r-- | glanceclient/v2/images.py | 87 | ||||
| -rw-r--r-- | glanceclient/v2/shell.py | 213 |
12 files changed, 1059 insertions, 104 deletions
diff --git a/glanceclient/common/http.py b/glanceclient/common/http.py index a5fb153..78c4bc5 100644 --- a/glanceclient/common/http.py +++ b/glanceclient/common/http.py @@ -66,7 +66,11 @@ def encode_headers(headers): for h, v in headers.items(): if v is not None: # if the item is token, do not quote '+' as well. - safe = '=+/' if h in TOKEN_HEADERS else '/' + # NOTE(imacdonn): urlparse.quote() is intended for quoting the + # path part of a URL, but headers like x-image-meta-location + # include an entire URL. We should avoid encoding the colon in + # this case (bug #1788942) + safe = '=+/' if h in TOKEN_HEADERS else '/:' if six.PY2: # incoming items may be unicode, so get them into something # the py2 version of urllib can handle before percent encoding @@ -179,6 +183,15 @@ class HTTPClient(_BaseHTTPClient): self.session.cert = (kwargs.get('cert_file'), kwargs.get('key_file')) + def __del__(self): + if self.session: + try: + self.session.close() + except Exception as e: + LOG.exception(e) + finally: + self.session = None + @staticmethod def parse_endpoint(endpoint): return netutils.urlsplit(endpoint) @@ -266,6 +279,7 @@ class HTTPClient(_BaseHTTPClient): conn_url, data=data, headers=headers, + timeout=self.timeout, **kwargs) except requests.exceptions.Timeout as e: message = ("Error communicating with %(url)s: %(e)s" % diff --git a/glanceclient/common/utils.py b/glanceclient/common/utils.py index dee9978..bc0c0eb 100644 --- a/glanceclient/common/utils.py +++ b/glanceclient/common/utils.py @@ -28,10 +28,10 @@ import uuid import six -if os.name == 'nt': - import msvcrt -else: - msvcrt = None +if os.name == 'nt': # noqa + import msvcrt # noqa +else: # noqa + msvcrt = None # noqa from oslo_utils import encodeutils from oslo_utils import strutils @@ -449,6 +449,26 @@ def integrity_iter(iter, checksum): (md5sum, checksum)) +def serious_integrity_iter(iter, hasher, hash_value): + """Check image data integrity using the Glance "multihash". + + :param iter: iterable containing the image data + :param hasher: a hashlib object + :param hash_value: hexdigest of the image data + :raises: IOError if the hashdigest of the data is not hash_value + """ + for chunk in iter: + yield chunk + if isinstance(chunk, six.string_types): + chunk = six.b(chunk) + hasher.update(chunk) + computed = hasher.hexdigest() + if computed != hash_value: + raise IOError(errno.EPIPE, + 'Corrupt image download. Hash was %s expected %s' % + (computed, hash_value)) + + def memoized_property(fn): attr_name = '_lazy_once_' + fn.__name__ diff --git a/glanceclient/exc.py b/glanceclient/exc.py index c8616c3..eee47ca 100644 --- a/glanceclient/exc.py +++ b/glanceclient/exc.py @@ -52,7 +52,7 @@ class HTTPException(ClientException): self.details = details or self.__class__.__name__ def __str__(self): - return "%s (HTTP %s)" % (self.details, self.code) + return "HTTP %s" % (self.details) class HTTPMultipleChoices(HTTPException): diff --git a/glanceclient/tests/unit/test_http.py b/glanceclient/tests/unit/test_http.py index 6eee005..2f72b9a 100644 --- a/glanceclient/tests/unit/test_http.py +++ b/glanceclient/tests/unit/test_http.py @@ -216,7 +216,11 @@ class TestClient(testtools.TestCase): def test_headers_encoding(self): value = u'ni\xf1o' - headers = {"test": value, "none-val": None, "Name": "value"} + fake_location = b'http://web_server:80/images/fake.img' + headers = {"test": value, + "none-val": None, + "Name": "value", + "x-image-meta-location": fake_location} encoded = http.encode_headers(headers) # Bug #1766235: According to RFC 8187, headers must be # encoded as 7-bit ASCII, so expect to see only displayable @@ -225,6 +229,8 @@ class TestClient(testtools.TestCase): self.assertNotIn("none-val", encoded) self.assertNotIn(b"none-val", encoded) self.assertEqual(b"value", encoded[b"Name"]) + # Bug #1788942: Colons in URL should not get percent-encoded + self.assertEqual(fake_location, encoded[b"x-image-meta-location"]) @mock.patch('keystoneauth1.adapter.Adapter.request') def test_http_duplicate_content_type_headers(self, mock_ksarq): diff --git a/glanceclient/tests/unit/test_shell.py b/glanceclient/tests/unit/test_shell.py index 0f15007..3027e46 100644 --- a/glanceclient/tests/unit/test_shell.py +++ b/glanceclient/tests/unit/test_shell.py @@ -965,7 +965,9 @@ class ShellTestRequests(testutils.TestCase): self.requests = self.useFixture(rm_fixture.Fixture()) self.requests.get('http://example.com/v2/images/%s/file' % id, headers=headers, raw=fake) - + self.requests.get('http://example.com/v2/images/%s' % id, + headers={'Content-type': 'application/json'}, + json=image_show_fixture) shell = openstack_shell.OpenStackImagesShell() argstr = ('--os-image-api-version 2 --os-auth-token faketoken ' '--os-image-url http://example.com ' diff --git a/glanceclient/tests/unit/v1/test_shell.py b/glanceclient/tests/unit/v1/test_shell.py index 95bbd07..a3bd29b 100644 --- a/glanceclient/tests/unit/v1/test_shell.py +++ b/glanceclient/tests/unit/v1/test_shell.py @@ -334,8 +334,8 @@ class ShellInvalidEndpointandParameterTest(utils.TestCase): e = self.assertRaises(exc.CommandError, self.run_command, '--os-image-api-version 1 image-create ' + origin + ' fake_src --container-format bare') - self.assertEqual('error: Must provide --disk-format when using ' - + origin + '.', e.message) + self.assertEqual('error: Must provide --disk-format when using ' + + origin + '.', e.message) @mock.patch('sys.stderr') def test_image_create_missing_container_format(self, __): @@ -536,8 +536,8 @@ class ShellStdinHandlingTests(testtools.TestCase): self._do_update() self.assertTrue( - 'data' not in self.collected_args[1] - or self.collected_args[1]['data'] is None + 'data' not in self.collected_args[1] or + self.collected_args[1]['data'] is None ) def test_image_update_opened_stdin(self): diff --git a/glanceclient/tests/unit/v2/fixtures.py b/glanceclient/tests/unit/v2/fixtures.py index 5a603c0..22e1ff7 100644 --- a/glanceclient/tests/unit/v2/fixtures.py +++ b/glanceclient/tests/unit/v2/fixtures.py @@ -14,6 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. +import hashlib + + UUID = "3fc2ba62-9a02-433e-b565-d493ffc69034" image_list_fixture = { @@ -65,7 +68,9 @@ image_show_fixture = { "tags": [], "updated_at": "2015-07-24T12:18:13Z", "virtual_size": "null", - "visibility": "shared" + "visibility": "shared", + "os_hash_algo": "sha384", + "os_hash_value": hashlib.sha384(b'DATA').hexdigest() } image_create_fixture = { diff --git a/glanceclient/tests/unit/v2/test_images.py b/glanceclient/tests/unit/v2/test_images.py index 23cbb43..99926de 100644 --- a/glanceclient/tests/unit/v2/test_images.py +++ b/glanceclient/tests/unit/v2/test_images.py @@ -14,6 +14,7 @@ # under the License. import errno +import hashlib import mock import testtools @@ -25,6 +26,10 @@ from glanceclient.v2 import images _CHKSUM = '93264c3edf5972c9f1cb309543d38a5c' _CHKSUM1 = '54264c3edf5972c9f1cb309453d38a46' +_HASHVAL = '54264c3edf93264c3edf5972c9f1cb309543d38a5c5972c9f1cb309453d38a46' +_HASHVAL1 = 'cb309543d38a5c5972c9f1cb309453d38a4654264c3edf93264c3edf5972c9f1' +_HASHBAD = '93264c3edf597254264c3edf5972c9f1cb309453d38a46c9f1cb309543d38a5c' + _TAG1 = 'power' _TAG2 = '64bit' @@ -193,7 +198,27 @@ data_fixtures = { 'A', ), }, - '/v2/images/66fb18d6-db27-11e1-a1eb-080027cbe205/file': { + '/v2/images/5cc4bebc-db27-11e1-a1eb-080027cbe205': { + 'GET': ( + {}, + {}, + ), + }, + '/v2/images/headeronly-db27-11e1-a1eb-080027cbe205/file': { + 'GET': ( + { + 'content-md5': 'wrong' + }, + 'BB', + ), + }, + '/v2/images/headeronly-db27-11e1-a1eb-080027cbe205': { + 'GET': ( + {}, + {}, + ), + }, + '/v2/images/chkonly-db27-11e1-a1eb-080027cbe205/file': { 'GET': ( { 'content-md5': 'wrong' @@ -201,7 +226,83 @@ data_fixtures = { 'BB', ), }, - '/v2/images/1b1c6366-dd57-11e1-af0f-02163e68b1d8/file': { + '/v2/images/chkonly-db27-11e1-a1eb-080027cbe205': { + 'GET': ( + {}, + { + 'checksum': 'wrong', + }, + ), + }, + '/v2/images/multihash-db27-11e1-a1eb-080027cbe205/file': { + 'GET': ( + { + 'content-md5': 'wrong' + }, + 'BB', + ), + }, + '/v2/images/multihash-db27-11e1-a1eb-080027cbe205': { + 'GET': ( + {}, + { + 'checksum': 'wrong', + 'os_hash_algo': 'md5', + 'os_hash_value': 'junk' + }, + ), + }, + '/v2/images/badalgo-db27-11e1-a1eb-080027cbe205/file': { + 'GET': ( + { + 'content-md5': hashlib.md5(b'BB').hexdigest() + }, + 'BB', + ), + }, + '/v2/images/badalgo-db27-11e1-a1eb-080027cbe205': { + 'GET': ( + {}, + { + 'checksum': hashlib.md5(b'BB').hexdigest(), + 'os_hash_algo': 'not_an_algo', + 'os_hash_value': 'whatever' + }, + ), + }, + '/v2/images/bad-multihash-value-good-checksum/file': { + 'GET': ( + { + 'content-md5': hashlib.md5(b'GOODCHECKSUM').hexdigest() + }, + 'GOODCHECKSUM', + ), + }, + '/v2/images/bad-multihash-value-good-checksum': { + 'GET': ( + {}, + { + 'checksum': hashlib.md5(b'GOODCHECKSUM').hexdigest(), + 'os_hash_algo': 'sha512', + 'os_hash_value': 'badmultihashvalue' + }, + ), + }, + '/v2/images/headeronly-dd57-11e1-af0f-02163e68b1d8/file': { + 'GET': ( + { + 'content-md5': 'defb99e69a9f1f6e06f15006b1f166ae' + }, + 'CCC', + ), + }, + '/v2/images/headeronly-dd57-11e1-af0f-02163e68b1d8': { + 'GET': ( + {}, + {}, + ), + }, + '/v2/images/chkonly-dd57-11e1-af0f-02163e68b1d8/file': { 'GET': ( { 'content-md5': 'defb99e69a9f1f6e06f15006b1f166ae' @@ -209,6 +310,32 @@ data_fixtures = { 'CCC', ), }, + '/v2/images/chkonly-dd57-11e1-af0f-02163e68b1d8': { + 'GET': ( + {}, + { + 'checksum': 'defb99e69a9f1f6e06f15006b1f166ae', + }, + ), + }, + '/v2/images/multihash-dd57-11e1-af0f-02163e68b1d8/file': { + 'GET': ( + { + 'content-md5': 'defb99e69a9f1f6e06f15006b1f166ae' + }, + 'CCC', + ), + }, + '/v2/images/multihash-dd57-11e1-af0f-02163e68b1d8': { + 'GET': ( + {}, + { + 'checksum': 'defb99e69a9f1f6e06f15006b1f166ae', + 'os_hash_algo': 'sha384', + 'os_hash_value': hashlib.sha384(b'CCC').hexdigest() + }, + ), + }, '/v2/images/87b634c1-f893-33c9-28a9-e5673c99239a/actions/reactivate': { 'POST': ({}, None) }, @@ -334,6 +461,41 @@ data_fixtures = { {'images': []}, ), }, + '/v2/images?limit=%d&os_hash_value=%s' % (images.DEFAULT_PAGE_SIZE, + _HASHVAL): { + 'GET': ( + {}, + {'images': [ + { + 'id': '3a4560a1-e585-443e-9b39-553b46ec92d1', + 'name': 'image-1', + } + ]}, + ), + }, + '/v2/images?limit=%d&os_hash_value=%s' % (images.DEFAULT_PAGE_SIZE, + _HASHVAL1): { + 'GET': ( + {}, + {'images': [ + { + 'id': '2a4560b2-e585-443e-9b39-553b46ec92d1', + 'name': 'image-1', + }, + { + 'id': '6f99bf80-2ee6-47cf-acfe-1f1fabb7e810', + 'name': 'image-2', + }, + ]}, + ), + }, + '/v2/images?limit=%d&os_hash_value=%s' % (images.DEFAULT_PAGE_SIZE, + _HASHBAD): { + 'GET': ( + {}, + {'images': []}, + ), + }, '/v2/images?limit=%d&tag=%s' % (images.DEFAULT_PAGE_SIZE, _TAG1): { 'GET': ( {}, @@ -631,6 +793,27 @@ class TestController(testtools.TestCase): images = self.controller.list(**filters) self.assertEqual(0, len(images)) + def test_list_images_for_hash_single_image(self): + fake_id = '3a4560a1-e585-443e-9b39-553b46ec92d1' + filters = {'filters': {'os_hash_value': _HASHVAL}} + images = self.controller.list(**filters) + self.assertEqual(1, len(images)) + self.assertEqual('%s' % fake_id, images[0].id) + + def test_list_images_for_hash_multiple_images(self): + fake_id1 = '2a4560b2-e585-443e-9b39-553b46ec92d1' + fake_id2 = '6f99bf80-2ee6-47cf-acfe-1f1fabb7e810' + filters = {'filters': {'os_hash_value': _HASHVAL1}} + images = self.controller.list(**filters) + self.assertEqual(2, len(images)) + self.assertEqual('%s' % fake_id1, images[0].id) + self.assertEqual('%s' % fake_id2, images[1].id) + + def test_list_images_for_wrong_hash(self): + filters = {'filters': {'os_hash_value': _HASHBAD}} + images = self.controller.list(**filters) + self.assertEqual(0, len(images)) + def test_list_images_for_bogus_owner(self): filters = {'filters': {'owner': _BOGUS_ID}} images = self.controller.list(**filters) @@ -846,12 +1029,24 @@ class TestController(testtools.TestCase): self.assertEqual('A', body) def test_data_with_wrong_checksum(self): - body = self.controller.data('66fb18d6-db27-11e1-a1eb-080027cbe205', + body = self.controller.data('headeronly-db27-11e1-a1eb-080027cbe205', do_checksum=False) body = ''.join([b for b in body]) self.assertEqual('BB', body) + body = self.controller.data('headeronly-db27-11e1-a1eb-080027cbe205') + try: + body = ''.join([b for b in body]) + self.fail('data did not raise an error.') + except IOError as e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'was 9d3d9048db16a7eee539e93e3618cbe7 expected wrong' + self.assertIn(msg, str(e)) - body = self.controller.data('66fb18d6-db27-11e1-a1eb-080027cbe205') + body = self.controller.data('chkonly-db27-11e1-a1eb-080027cbe205', + do_checksum=False) + body = ''.join([b for b in body]) + self.assertEqual('BB', body) + body = self.controller.data('chkonly-db27-11e1-a1eb-080027cbe205') try: body = ''.join([b for b in body]) self.fail('data did not raise an error.') @@ -860,15 +1055,103 @@ class TestController(testtools.TestCase): msg = 'was 9d3d9048db16a7eee539e93e3618cbe7 expected wrong' self.assertIn(msg, str(e)) - def test_data_with_checksum(self): - body = self.controller.data('1b1c6366-dd57-11e1-af0f-02163e68b1d8', + body = self.controller.data('multihash-db27-11e1-a1eb-080027cbe205', do_checksum=False) body = ''.join([b for b in body]) - self.assertEqual('CCC', body) + self.assertEqual('BB', body) + body = self.controller.data('multihash-db27-11e1-a1eb-080027cbe205') + try: + body = ''.join([b for b in body]) + self.fail('data did not raise an error.') + except IOError as e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'was 9d3d9048db16a7eee539e93e3618cbe7 expected junk' + self.assertIn(msg, str(e)) + + body = self.controller.data('badalgo-db27-11e1-a1eb-080027cbe205', + do_checksum=False) + body = ''.join([b for b in body]) + self.assertEqual('BB', body) + try: + body = self.controller.data('badalgo-db27-11e1-a1eb-080027cbe205') + self.fail('bad os_hash_algo did not raise an error.') + except ValueError as e: + msg = 'unsupported hash type not_an_algo' + self.assertIn(msg, str(e)) + + def test_data_with_checksum(self): + for prefix in ['headeronly', 'chkonly', 'multihash']: + body = self.controller.data(prefix + + '-dd57-11e1-af0f-02163e68b1d8', + do_checksum=False) + body = ''.join([b for b in body]) + self.assertEqual('CCC', body) + + body = self.controller.data(prefix + + '-dd57-11e1-af0f-02163e68b1d8') + body = ''.join([b for b in body]) + self.assertEqual('CCC', body) + + def test_data_with_checksum_and_fallback(self): + # make sure the allow_md5_fallback option does not cause any + # incorrect behavior when fallback is not needed + for prefix in ['headeronly', 'chkonly', 'multihash']: + body = self.controller.data(prefix + + '-dd57-11e1-af0f-02163e68b1d8', + do_checksum=False, + allow_md5_fallback=True) + body = ''.join([b for b in body]) + self.assertEqual('CCC', body) + + body = self.controller.data(prefix + + '-dd57-11e1-af0f-02163e68b1d8', + allow_md5_fallback=True) + body = ''.join([b for b in body]) + self.assertEqual('CCC', body) + + def test_data_with_bad_hash_algo_and_fallback(self): + # shouldn't matter when do_checksum is False + body = self.controller.data('badalgo-db27-11e1-a1eb-080027cbe205', + do_checksum=False, + allow_md5_fallback=True) + body = ''.join([b for b in body]) + self.assertEqual('BB', body) + + # default value for do_checksum is True + body = self.controller.data('badalgo-db27-11e1-a1eb-080027cbe205', + allow_md5_fallback=True) + body = ''.join([b for b in body]) + self.assertEqual('BB', body) - body = self.controller.data('1b1c6366-dd57-11e1-af0f-02163e68b1d8') + def test_neg_data_with_bad_hash_value_and_fallback_enabled(self): + # make sure download fails when good hash_algo but bad hash_value + # even when correct checksum is present regardless of + # allow_md5_fallback setting + body = self.controller.data('bad-multihash-value-good-checksum', + allow_md5_fallback=False) + try: + body = ''.join([b for b in body]) + self.fail('bad os_hash_value did not raise an error.') + except IOError as e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'expected badmultihashvalue' + self.assertIn(msg, str(e)) + + body = self.controller.data('bad-multihash-value-good-checksum', + allow_md5_fallback=True) + try: + body = ''.join([b for b in body]) + self.fail('bad os_hash_value did not raise an error.') + except IOError as e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'expected badmultihashvalue' + self.assertIn(msg, str(e)) + + # download should succeed when do_checksum is off, though + body = self.controller.data('bad-multihash-value-good-checksum', + do_checksum=False) body = ''.join([b for b in body]) - self.assertEqual('CCC', body) + self.assertEqual('GOODCHECKSUM', body) def test_image_import(self): uri = 'http://example.com/image.qcow' @@ -883,7 +1166,7 @@ class TestController(testtools.TestCase): def test_download_no_data(self): resp = utils.FakeResponse(headers={}, status_code=204) self.controller.controller.http_client.get = mock.Mock( - return_value=(resp, None)) + return_value=(resp, {})) self.controller.data('image_id') def test_download_forbidden(self): diff --git a/glanceclient/tests/unit/v2/test_shell_v2.py b/glanceclient/tests/unit/v2/test_shell_v2.py index 6eeca83..c43f606 100644 --- a/glanceclient/tests/unit/v2/test_shell_v2.py +++ b/glanceclient/tests/unit/v2/test_shell_v2.py @@ -57,7 +57,7 @@ def schema_args(schema_getter, omit=None): return original_schema_args(my_schema_getter, omit) utils.schema_args = schema_args -from glanceclient.v2 import shell as test_shell +from glanceclient.v2 import shell as test_shell # noqa # Return original decorator. utils.schema_args = original_schema_args @@ -95,7 +95,7 @@ class ShellV2Test(testtools.TestCase): # dict directly, it throws an AttributeError. class Args(object): def __init__(self, entries): - self.backend = None + self.store = None self.__dict__.update(entries) return Args(args) @@ -264,6 +264,8 @@ class ShellV2Test(testtools.TestCase): 'sort_dir': ['desc', 'asc'], 'sort': None, 'verbose': False, + 'include_stores': False, + 'os_hash_value': None, 'os_hidden': False } args = self._make_args(input) @@ -286,6 +288,86 @@ class ShellV2Test(testtools.TestCase): filters=exp_img_filters) utils.print_list.assert_called_once_with({}, ['ID', 'Name']) + def test_do_image_list_verbose(self): + input = { + 'limit': None, + 'page_size': 18, + 'visibility': True, + 'member_status': 'Fake', + 'owner': 'test', + 'checksum': 'fake_checksum', + 'tag': 'fake tag', + 'properties': [], + 'sort_key': ['name', 'id'], + 'sort_dir': ['desc', 'asc'], + 'sort': None, + 'verbose': True, + 'include_stores': False, + 'os_hash_value': None, + 'os_hidden': False + } + args = self._make_args(input) + with mock.patch.object(self.gc.images, 'list') as mocked_list: + mocked_list.return_value = {} + + test_shell.do_image_list(self.gc, args) + utils.print_list.assert_called_once_with( + {}, ['ID', 'Name', 'Disk_format', 'Container_format', + 'Size', 'Status', 'Owner']) + + def test_do_image_list_with_include_stores_true(self): + input = { + 'limit': None, + 'page_size': 18, + 'visibility': True, + 'member_status': 'Fake', + 'owner': 'test', + 'checksum': 'fake_checksum', + 'tag': 'fake tag', + 'properties': [], + 'sort_key': ['name', 'id'], + 'sort_dir': ['desc', 'asc'], + 'sort': None, + 'verbose': False, + 'include_stores': True, + 'os_hash_value': None, + 'os_hidden': False + } + args = self._make_args(input) + with mock.patch.object(self.gc.images, 'list') as mocked_list: + mocked_list.return_value = {} + + test_shell.do_image_list(self.gc, args) + utils.print_list.assert_called_once_with( + {}, ['ID', 'Name', 'Stores']) + + def test_do_image_list_verbose_with_include_stores_true(self): + input = { + 'limit': None, + 'page_size': 18, + 'visibility': True, + 'member_status': 'Fake', + 'owner': 'test', + 'checksum': 'fake_checksum', + 'tag': 'fake tag', + 'properties': [], + 'sort_key': ['name', 'id'], + 'sort_dir': ['desc', 'asc'], + 'sort': None, + 'verbose': True, + 'include_stores': True, + 'os_hash_value': None, + 'os_hidden': False + } + args = self._make_args(input) + with mock.patch.object(self.gc.images, 'list') as mocked_list: + mocked_list.return_value = {} + + test_shell.do_image_list(self.gc, args) + utils.print_list.assert_called_once_with( + {}, ['ID', 'Name', 'Disk_format', 'Container_format', + 'Size', 'Status', 'Owner', 'Stores']) + def test_do_image_list_with_hidden_true(self): input = { 'limit': None, @@ -300,6 +382,8 @@ class ShellV2Test(testtools.TestCase): 'sort_dir': ['desc', 'asc'], 'sort': None, 'verbose': False, + 'include_stores': False, + 'os_hash_value': None, 'os_hidden': True } args = self._make_args(input) @@ -336,6 +420,8 @@ class ShellV2Test(testtools.TestCase): 'sort_dir': ['desc'], 'sort': None, 'verbose': False, + 'include_stores': False, + 'os_hash_value': None, 'os_hidden': False } args = self._make_args(input) @@ -372,6 +458,8 @@ class ShellV2Test(testtools.TestCase): 'sort_key': [], 'sort_dir': [], 'verbose': False, + 'include_stores': False, + 'os_hash_value': None, 'os_hidden': False } args = self._make_args(input) @@ -408,6 +496,8 @@ class ShellV2Test(testtools.TestCase): 'sort_dir': ['desc'], 'sort': None, 'verbose': False, + 'include_stores': False, + 'os_hash_value': None, 'os_hidden': False } args = self._make_args(input) @@ -692,9 +782,9 @@ class ShellV2Test(testtools.TestCase): @mock.patch('glanceclient.common.utils.exit') @mock.patch('os.access') @mock.patch('sys.stdin', autospec=True) - def test_neg_do_image_create_no_file_and_stdin_with_backend( + def test_neg_do_image_create_no_file_and_stdin_with_store( self, mock_stdin, mock_access, mock_utils_exit): - expected_msg = ('--backend option should only be provided with --file ' + expected_msg = ('--store option should only be provided with --file ' 'option or stdin.') mock_utils_exit.side_effect = self._mock_utils_exit mock_stdin.isatty = lambda: True @@ -702,7 +792,7 @@ class ShellV2Test(testtools.TestCase): args = self._make_args({'name': 'IMG-01', 'property': ['myprop=myval'], 'file': None, - 'backend': 'file1', + 'store': 'file1', 'container_format': 'bare', 'disk_format': 'qcow2'}) @@ -715,16 +805,16 @@ class ShellV2Test(testtools.TestCase): mock_utils_exit.assert_called_once_with(expected_msg) @mock.patch('glanceclient.common.utils.exit') - def test_neg_do_image_create_invalid_backend( + def test_neg_do_image_create_invalid_store( self, mock_utils_exit): - expected_msg = ("Backend 'dummy' is not valid for this cloud. " + expected_msg = ("Store 'dummy' is not valid for this cloud. " "Valid values can be retrieved with stores-info " "command.") mock_utils_exit.side_effect = self._mock_utils_exit args = self._make_args({'name': 'IMG-01', 'property': ['myprop=myval'], 'file': "somefile.txt", - 'backend': 'dummy', + 'store': 'dummy', 'container_format': 'bare', 'disk_format': 'qcow2'}) @@ -752,7 +842,7 @@ class ShellV2Test(testtools.TestCase): import_info_response = {'import-methods': { 'type': 'array', 'description': 'Import methods available.', - 'value': ['glance-direct', 'web-download']}} + 'value': ['glance-direct', 'web-download', 'copy-image']}} def _mock_utils_exit(self, msg): sys.exit(msg) @@ -781,15 +871,120 @@ class ShellV2Test(testtools.TestCase): mock_utils_exit.assert_called_once_with(expected_msg) @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_copy_image( + self, mock_utils_exit): + expected_msg = ("Import method 'copy-image' cannot be used " + "while creating the image.") + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'copy-image'}) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_stores_all_stores_specified( + self, mock_utils_exit): + expected_msg = ('Only one of --store, --stores and --all-stores can ' + 'be provided') + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'glance-direct', + 'stores': 'file1,file2', 'os_all_stores': True, + 'file': 'some.mufile', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + @mock.patch('sys.stdin', autospec=True) + def test_neg_image_create_via_import_stores_without_file( + self, mock_stdin, mock_utils_exit): + expected_msg = ('--stores option should only be provided with --file ' + 'option or stdin for the glance-direct import method.') + mock_utils_exit.side_effect = self._mock_utils_exit + mock_stdin.isatty = lambda: True + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'glance-direct', + 'stores': 'file1,file2', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + with mock.patch.object(self.gc.images, + 'get_stores_info') as mocked_stores_info: + mocked_stores_info.return_value = self.stores_info_response + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + @mock.patch('sys.stdin', autospec=True) + def test_neg_image_create_via_import_all_stores_without_file( + self, mock_stdin, mock_utils_exit): + expected_msg = ('--all-stores option should only be provided with ' + '--file option or stdin for the glance-direct import ' + 'method.') + mock_utils_exit.side_effect = self._mock_utils_exit + mock_stdin.isatty = lambda: True + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'glance-direct', + 'os_all_stores': True, + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') @mock.patch('os.access') @mock.patch('sys.stdin', autospec=True) - def test_neg_image_create_via_import_no_file_and_stdin_with_backend( + def test_neg_image_create_via_import_no_file_and_stdin_with_store( self, mock_stdin, mock_access, mock_utils_exit): - expected_msg = ('--backend option should only be provided with --file ' + expected_msg = ('--store option should only be provided with --file ' 'option or stdin for the glance-direct import method.') my_args = self.base_args.copy() my_args['import_method'] = 'glance-direct' - my_args['backend'] = 'file1' + my_args['store'] = 'file1' args = self._make_args(my_args) mock_stdin.isatty = lambda: True @@ -810,13 +1005,13 @@ class ShellV2Test(testtools.TestCase): @mock.patch('glanceclient.common.utils.exit') @mock.patch('sys.stdin', autospec=True) - def test_neg_image_create_via_import_no_uri_with_backend( + def test_neg_image_create_via_import_no_uri_with_store( self, mock_stdin, mock_utils_exit): - expected_msg = ('--backend option should only be provided with --uri ' + expected_msg = ('--store option should only be provided with --uri ' 'option for the web-download import method.') my_args = self.base_args.copy() my_args['import_method'] = 'web-download' - my_args['backend'] = 'file1' + my_args['store'] = 'file1' args = self._make_args(my_args) mock_utils_exit.side_effect = self._mock_utils_exit with mock.patch.object(self.gc.images, @@ -835,14 +1030,14 @@ class ShellV2Test(testtools.TestCase): @mock.patch('glanceclient.common.utils.exit') @mock.patch('os.access') @mock.patch('sys.stdin', autospec=True) - def test_neg_image_create_via_import_invalid_backend( + def test_neg_image_create_via_import_invalid_store( self, mock_stdin, mock_access, mock_utils_exit): - expected_msg = ("Backend 'dummy' is not valid for this cloud. " + expected_msg = ("Store 'dummy' is not valid for this cloud. " "Valid values can be retrieved with stores-info" " command.") my_args = self.base_args.copy() my_args['import_method'] = 'glance-direct' - my_args['backend'] = 'dummy' + my_args['store'] = 'dummy' args = self._make_args(my_args) mock_stdin.isatty = lambda: True @@ -994,6 +1189,60 @@ class ShellV2Test(testtools.TestCase): mock_utils_exit.assert_called_once_with(expected_msg) @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_stores_without_uri( + self, mock_utils_exit): + expected_msg = ('--stores option should only be provided with --uri ' + 'option for the web-download import method.') + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'web-download', + 'stores': 'file1,file2', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + with mock.patch.object(self.gc.images, + 'get_stores_info') as mocked_stores_info: + mocked_stores_info.return_value = self.stores_info_response + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_all_stores_without_uri( + self, mock_utils_exit): + expected_msg = ('--all-stores option should only be provided with ' + '--uri option for the web-download import ' + 'method.') + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'web-download', + 'os_all_stores': True, + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') @mock.patch('sys.stdin', autospec=True) def test_neg_image_create_via_import_web_download_no_uri_with_file( self, mock_stdin, mock_utils_exit): @@ -1428,18 +1677,25 @@ class ShellV2Test(testtools.TestCase): def test_do_location_add(self): gc = self.gc - loc = {'url': 'http://foo.com/', 'metadata': {'foo': 'bar'}} - args = self._make_args({'id': 'pass', - 'url': loc['url'], - 'metadata': json.dumps(loc['metadata'])}) + loc = {'url': 'http://foo.com/', + 'metadata': {'foo': 'bar'}, + 'validation_data': {'checksum': 'csum', + 'os_hash_algo': 'algo', + 'os_hash_value': 'value'}} + args = {'id': 'pass', + 'url': loc['url'], + 'metadata': json.dumps(loc['metadata']), + 'checksum': 'csum', + 'hash_algo': 'algo', + 'hash_value': 'value'} with mock.patch.object(gc.images, 'add_location') as mocked_addloc: expect_image = {'id': 'pass', 'locations': [loc]} mocked_addloc.return_value = expect_image - test_shell.do_location_add(self.gc, args) - mocked_addloc.assert_called_once_with('pass', - loc['url'], - loc['metadata']) + test_shell.do_location_add(self.gc, self._make_args(args)) + mocked_addloc.assert_called_once_with( + 'pass', loc['url'], loc['metadata'], + validation_data=loc['validation_data']) utils.print_dict.assert_called_once_with(expect_image) def test_do_location_delete(self): @@ -1479,15 +1735,15 @@ class ShellV2Test(testtools.TestCase): backend=None) @mock.patch('glanceclient.common.utils.exit') - def test_image_upload_invalid_backend(self, mock_utils_exit): - expected_msg = ("Backend 'dummy' is not valid for this cloud. " + def test_image_upload_invalid_store(self, mock_utils_exit): + expected_msg = ("Store 'dummy' is not valid for this cloud. " "Valid values can be retrieved with stores-info " "command.") mock_utils_exit.side_effect = self._mock_utils_exit args = self._make_args( {'id': 'IMG-01', 'file': 'test', 'size': 1024, 'progress': False, - 'backend': 'dummy'}) + 'store': 'dummy'}) with mock.patch.object(self.gc.images, 'get_stores_info') as mock_stores_info: @@ -1646,15 +1902,15 @@ class ShellV2Test(testtools.TestCase): mock_utils_exit.assert_called_once_with(expected_msg) @mock.patch('glanceclient.common.utils.exit') - def test_image_import_invalid_backend(self, mock_utils_exit): - expected_msg = ("Backend 'dummy' is not valid for this cloud. " + def test_image_import_invalid_store(self, mock_utils_exit): + expected_msg = ("Store 'dummy' is not valid for this cloud. " "Valid values can be retrieved with stores-info " "command.") mock_utils_exit.side_effect = self._mock_utils_exit args = self._make_args( {'id': 'IMG-01', 'import_method': 'glance-direct', 'uri': None, - 'backend': 'dummy'}) + 'store': 'dummy'}) with mock.patch.object(self.gc.images, 'get') as mocked_get: with mock.patch.object(self.gc.images, @@ -1688,7 +1944,8 @@ class ShellV2Test(testtools.TestCase): mock_import.return_value = None test_shell.do_image_import(self.gc, args) mock_import.assert_called_once_with( - 'IMG-01', 'glance-direct', None, backend=None) + 'IMG-01', 'glance-direct', None, backend=None, + all_stores=None, allow_failure=True, stores=None) def test_image_import_web_download(self): args = self._make_args( @@ -1706,7 +1963,9 @@ class ShellV2Test(testtools.TestCase): test_shell.do_image_import(self.gc, args) mock_import.assert_called_once_with( 'IMG-01', 'web-download', - 'http://example.com/image.qcow', backend=None) + 'http://example.com/image.qcow', + all_stores=None, allow_failure=True, + backend=None, stores=None) @mock.patch('glanceclient.common.utils.print_image') def test_image_import_no_print_image(self, mocked_utils_print_image): @@ -1724,12 +1983,112 @@ class ShellV2Test(testtools.TestCase): mock_import.return_value = None test_shell.do_image_import(self.gc, args) mock_import.assert_called_once_with( - 'IMG-02', 'glance-direct', None, backend=None) + 'IMG-02', 'glance-direct', None, stores=None, + all_stores=None, allow_failure=True, backend=None) mocked_utils_print_image.assert_not_called() + @mock.patch('glanceclient.common.utils.print_image') + @mock.patch('glanceclient.v2.shell._validate_backend') + def test_image_import_multiple_stores(self, mocked_utils_print_image, + msvb): + args = self._make_args( + {'id': 'IMG-02', 'uri': None, 'import_method': 'glance-direct', + 'from_create': False, 'stores': 'site1,site2'}) + with mock.patch.object(self.gc.images, 'image_import') as mock_import: + with mock.patch.object(self.gc.images, 'get') as mocked_get: + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_get.return_value = {'status': 'uploading', + 'container_format': 'bare', + 'disk_format': 'raw'} + mocked_info.return_value = self.import_info_response + mock_import.return_value = None + test_shell.do_image_import(self.gc, args) + mock_import.assert_called_once_with( + 'IMG-02', 'glance-direct', None, all_stores=None, + allow_failure=True, stores=['site1', 'site2'], + backend=None) + + @mock.patch('glanceclient.common.utils.print_image') + @mock.patch('glanceclient.v2.shell._validate_backend') + def test_image_import_copy_image(self, mocked_utils_print_image, + msvb): + args = self._make_args( + {'id': 'IMG-02', 'uri': None, 'import_method': 'copy-image', + 'from_create': False, 'stores': 'file1,file2'}) + with mock.patch.object(self.gc.images, 'image_import') as mock_import: + with mock.patch.object(self.gc.images, 'get') as mocked_get: + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_get.return_value = {'status': 'active', + 'container_format': 'bare', + 'disk_format': 'raw'} + mocked_info.return_value = self.import_info_response + mock_import.return_value = None + test_shell.do_image_import(self.gc, args) + mock_import.assert_called_once_with( + 'IMG-02', 'copy-image', None, all_stores=None, + allow_failure=True, stores=['file1', 'file2'], + backend=None) + + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_import_copy_image_not_active( + self, mock_utils_exit): + expected_msg = ("The 'copy-image' import method can only be used on " + "an image with status 'active'.") + mock_utils_exit.side_effect = self._mock_utils_exit + args = self._make_args( + {'id': 'IMG-02', 'uri': None, 'import_method': 'copy-image', + 'disk_format': 'raw', + 'container_format': 'bare', + 'from_create': False, 'stores': 'file1,file2'}) + with mock.patch.object( + self.gc.images, + 'get_stores_info') as mocked_stores_info: + with mock.patch.object(self.gc.images, 'get') as mocked_get: + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + + mocked_stores_info.return_value = self.stores_info_response + mocked_get.return_value = {'status': 'uploading', + 'container_format': 'bare', + 'disk_format': 'raw'} + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_import_stores_all_stores_not_specified( + self, mock_utils_exit): + expected_msg = ("Provide either --stores or --all-stores for " + "'copy-image' import method.") + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'copy-image', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + def test_image_download(self): args = self._make_args( - {'id': 'IMG-01', 'file': 'test', 'progress': True}) + {'id': 'IMG-01', 'file': 'test', 'progress': True, + 'allow_md5_fallback': False}) with mock.patch.object(self.gc.images, 'data') as mocked_data, \ mock.patch.object(utils, '_extract_request_id'): @@ -1737,14 +2096,27 @@ class ShellV2Test(testtools.TestCase): [c for c in 'abcdef']) test_shell.do_image_download(self.gc, args) - mocked_data.assert_called_once_with('IMG-01') + mocked_data.assert_called_once_with('IMG-01', + allow_md5_fallback=False) + + # check that non-default value is being passed correctly + args.allow_md5_fallback = True + with mock.patch.object(self.gc.images, 'data') as mocked_data, \ + mock.patch.object(utils, '_extract_request_id'): + mocked_data.return_value = utils.RequestIdProxy( + [c for c in 'abcdef']) + + test_shell.do_image_download(self.gc, args) + mocked_data.assert_called_once_with('IMG-01', + allow_md5_fallback=True) @mock.patch.object(utils, 'exit') @mock.patch('sys.stdout', autospec=True) def test_image_download_no_file_arg(self, mocked_stdout, mocked_utils_exit): # Indicate that no file name was given as command line argument - args = self._make_args({'id': '1234', 'file': None, 'progress': False}) + args = self._make_args({'id': '1234', 'file': None, 'progress': False, + 'allow_md5_fallback': False}) # Indicate that no file is specified for output redirection mocked_stdout.isatty = lambda: True test_shell.do_image_download(self.gc, args) @@ -1794,6 +2166,29 @@ class ShellV2Test(testtools.TestCase): mocked_utils_exit.assert_called_once_with() @mock.patch.object(utils, 'exit') + def test_do_image_delete_from_store_not_found(self, mocked_utils_exit): + args = argparse.Namespace(id='image1', store='store1') + with mock.patch.object(self.gc.images, + 'delete_from_store') as mocked_delete: + mocked_delete.side_effect = exc.HTTPNotFound + + test_shell.do_stores_delete(self.gc, args) + + self.assertEqual(1, mocked_delete.call_count) + mocked_utils_exit.assert_called_once_with('Multi Backend support ' + 'is not enabled or ' + 'Image/store not found.') + + def test_do_image_delete_from_store(self): + args = argparse.Namespace(id='image1', store='store1') + with mock.patch.object(self.gc.images, + 'delete_from_store') as mocked_delete: + test_shell.do_stores_delete(self.gc, args) + + mocked_delete.assert_called_once_with('store1', + 'image1') + + @mock.patch.object(utils, 'exit') @mock.patch.object(utils, 'print_err') def test_do_image_delete_with_forbidden_ids(self, mocked_print_err, mocked_utils_exit): @@ -1835,7 +2230,8 @@ class ShellV2Test(testtools.TestCase): def test_do_image_download_with_forbidden_id(self, mocked_print_err, mocked_stdout): args = self._make_args({'id': 'IMG-01', 'file': None, - 'progress': False}) + 'progress': False, + 'allow_md5_fallback': False}) mocked_stdout.isatty = lambda: False with mock.patch.object(self.gc.images, 'data') as mocked_data: mocked_data.side_effect = exc.HTTPForbidden @@ -1852,7 +2248,8 @@ class ShellV2Test(testtools.TestCase): @mock.patch.object(utils, 'print_err') def test_do_image_download_with_500(self, mocked_print_err, mocked_stdout): args = self._make_args({'id': 'IMG-01', 'file': None, - 'progress': False}) + 'progress': False, + 'allow_md5_fallback': False}) mocked_stdout.isatty = lambda: False with mock.patch.object(self.gc.images, 'data') as mocked_data: mocked_data.side_effect = exc.HTTPInternalServerError diff --git a/glanceclient/v1/shell.py b/glanceclient/v1/shell.py index fff7490..8a4d29d 100644 --- a/glanceclient/v1/shell.py +++ b/glanceclient/v1/shell.py @@ -30,7 +30,7 @@ import glanceclient.v1.images CONTAINER_FORMATS = ('Acceptable formats: ami, ari, aki, bare, ovf, ova,' 'docker.') -DISK_FORMATS = ('Acceptable formats: ami, ari, aki, vhd, vdhx, vmdk, raw, ' +DISK_FORMATS = ('Acceptable formats: ami, ari, aki, vhd, vhdx, vmdk, raw, ' 'qcow2, vdi, iso, and ploop.') DATA_FIELDS = ('location', 'copy_from', 'file') @@ -45,11 +45,11 @@ _bool_strict = functools.partial(strutils.bool_from_string, strict=True) help='Filter images to those that changed since the given time' ', which will include the deleted images.') @utils.arg('--container-format', metavar='<CONTAINER_FORMAT>', - help='Filter images to those that have this container format. ' - + CONTAINER_FORMATS) + help='Filter images to those that have this container format. ' + + CONTAINER_FORMATS) @utils.arg('--disk-format', metavar='<DISK_FORMAT>', - help='Filter images to those that have this disk format. ' - + DISK_FORMATS) + help='Filter images to those that have this disk format. ' + + DISK_FORMATS) @utils.arg('--size-min', metavar='<SIZE>', type=int, help='Filter images to those with a size greater than this.') @utils.arg('--size-max', metavar='<SIZE>', type=int, diff --git a/glanceclient/v2/images.py b/glanceclient/v2/images.py index be804a2..1e8e621 100644 --- a/glanceclient/v2/images.py +++ b/glanceclient/v2/images.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import hashlib import json from oslo_utils import encodeutils from requests import codes @@ -197,13 +198,39 @@ class Controller(object): return self._get(image_id) @utils.add_req_id_to_object() - def data(self, image_id, do_checksum=True): + def data(self, image_id, do_checksum=True, allow_md5_fallback=False): """Retrieve data of an image. - :param image_id: ID of the image to download. - :param do_checksum: Enable/disable checksum validation. - :returns: An iterable body or None + When do_checksum is enabled, validation proceeds as follows: + + 1. if the image has a 'os_hash_value' property, the algorithm + specified in the image's 'os_hash_algo' property will be used + to validate against the 'os_hash_value' value. If the + specified hash algorithm is not available AND allow_md5_fallback + is True, then continue to step #2 + 2. else if the image has a checksum property, MD5 is used to + validate against the 'checksum' value + 3. else if the download response has a 'content-md5' header, MD5 + is used to validate against the header value + 4. if none of 1-3 obtain, the data is **not validated** (this is + compatible with legacy behavior) + + :param image_id: ID of the image to download + :param do_checksum: Enable/disable checksum validation + :param allow_md5_fallback: + Use the MD5 checksum for validation if the algorithm specified by + the image's 'os_hash_algo' property is not available + :returns: An iterable body or ``None`` """ + if do_checksum: + # doing this first to prevent race condition if image record + # is deleted during the image download + url = '/v2/images/%s' % image_id + resp, image_meta = self.http_client.get(url) + meta_checksum = image_meta.get('checksum', None) + meta_hash_value = image_meta.get('os_hash_value', None) + meta_hash_algo = image_meta.get('os_hash_algo', None) + url = '/v2/images/%s/file' % image_id resp, body = self.http_client.get(url) if resp.status_code == codes.no_content: @@ -212,8 +239,32 @@ class Controller(object): checksum = resp.headers.get('content-md5', None) content_length = int(resp.headers.get('content-length', 0)) - if do_checksum and checksum is not None: - body = utils.integrity_iter(body, checksum) + check_md5sum = do_checksum + if do_checksum and meta_hash_value is not None: + try: + hasher = hashlib.new(str(meta_hash_algo)) + body = utils.serious_integrity_iter(body, + hasher, + meta_hash_value) + check_md5sum = False + except ValueError as ve: + if (str(ve).startswith('unsupported hash type') and + allow_md5_fallback): + check_md5sum = True + else: + raise + + if do_checksum and check_md5sum: + if meta_checksum is not None: + body = utils.integrity_iter(body, meta_checksum) + elif checksum is not None: + body = utils.integrity_iter(body, checksum) + else: + # NOTE(rosmaita): this preserves legacy behavior to return the + # image data when checksumming is requested but there's no + # 'content-md5' header in the response. Just want to make it + # clear that we're doing this on purpose. + pass return utils.IterableWithLength(body, content_length), resp @@ -252,6 +303,14 @@ class Controller(object): return body, resp @utils.add_req_id_to_object() + def delete_from_store(self, store_id, image_id): + """Delete image data from specific store.""" + url = ('/v2/stores/%(store)s/%(image)s' % {'store': store_id, + 'image': image_id}) + resp, body = self.http_client.delete(url) + return body, resp + + @utils.add_req_id_to_object() def stage(self, image_id, image_data, image_size=None): """Upload the data to image staging. @@ -267,13 +326,22 @@ class Controller(object): @utils.add_req_id_to_object() def image_import(self, image_id, method='glance-direct', uri=None, - backend=None): + backend=None, stores=None, allow_failure=True, + all_stores=None): """Import Image via method.""" headers = {} url = '/v2/images/%s/import' % image_id data = {'method': {'name': method}} + if stores: + data['stores'] = stores + if allow_failure: + data['all_stores_must_succeed'] = 'false' if backend is not None: headers['x-image-meta-store'] = backend + if all_stores: + data['all_stores'] = 'true' + if allow_failure: + data['all_stores_must_succeed'] = 'false' if uri: if method == 'web-download': @@ -381,7 +449,7 @@ class Controller(object): data=json.dumps(patch_body)) return (resp, body), resp - def add_location(self, image_id, url, metadata): + def add_location(self, image_id, url, metadata, validation_data=None): """Add a new location entry to an image's list of locations. It is an error to add a URL that is already present in the list of @@ -390,10 +458,13 @@ class Controller(object): :param image_id: ID of image to which the location is to be added. :param url: URL of the location to add. :param metadata: Metadata associated with the location. + :param validation_data: Validation data for the image. :returns: The updated image """ add_patch = [{'op': 'add', 'path': '/locations/-', 'value': {'url': url, 'metadata': metadata}}] + if validation_data: + add_patch[0]['value']['validation_data'] = validation_data response = self._send_image_update_request(image_id, add_patch) # Get request id from the above update request and pass the same to # following get request diff --git a/glanceclient/v2/shell.py b/glanceclient/v2/shell.py index aaa85bb..b4dc811 100644 --- a/glanceclient/v2/shell.py +++ b/glanceclient/v2/shell.py @@ -71,8 +71,8 @@ def get_image_schema(): 'passed to the client via stdin.')) @utils.arg('--progress', action='store_true', default=False, help=_('Show upload progress bar.')) -@utils.arg('--backend', metavar='<STORE>', - default=utils.env('OS_IMAGE_BACKEND', default=None), +@utils.arg('--store', metavar='<STORE>', + default=utils.env('OS_IMAGE_STORE', default=None), help='Backend store to upload image to.') @utils.on_data_require_fields(DATA_FIELDS) def do_image_create(gc, args): @@ -90,12 +90,12 @@ def do_image_create(gc, args): key, value = datum.split('=', 1) fields[key] = value - backend = args.backend + backend = args.store file_name = fields.pop('file', None) using_stdin = not sys.stdin.isatty() - if args.backend and not (file_name or using_stdin): - utils.exit("--backend option should only be provided with --file " + if args.store and not (file_name or using_stdin): + utils.exit("--store option should only be provided with --file " "option or stdin.") if backend: @@ -108,7 +108,7 @@ def do_image_create(gc, args): image = gc.images.create(**fields) try: if utils.get_data_file(args) is not None: - backend = fields.get('backend', None) + backend = fields.get('store', None) args.id = image['id'] args.size = None do_image_upload(gc, args) @@ -147,9 +147,31 @@ def do_image_create(gc, args): 'record if no import-method and no data is supplied')) @utils.arg('--uri', metavar='<IMAGE_URL>', default=None, help=_('URI to download the external image.')) -@utils.arg('--backend', metavar='<STORE>', - default=utils.env('OS_IMAGE_BACKEND', default=None), +@utils.arg('--store', metavar='<STORE>', + default=utils.env('OS_IMAGE_STORE', default=None), help='Backend store to upload image to.') +@utils.arg('--stores', metavar='<STORES>', + default=utils.env('OS_IMAGE_STORES', default=None), + help=_('Stores to upload image to if multi-stores import ' + 'available. Comma separated list. Available stores can be ' + 'listed with "stores-info" call.')) +@utils.arg('--all-stores', type=strutils.bool_from_string, + metavar='[True|False]', + default=None, + dest='os_all_stores', + help=_('"all-stores" can be ued instead of "stores"-list to ' + 'indicate that image should be imported into all available ' + 'stores.')) +@utils.arg('--allow-failure', type=strutils.bool_from_string, + metavar='[True|False]', + dest='os_allow_failure', + default=utils.env('OS_IMAGE_ALLOW_FAILURE', default=True), + help=_('Indicator if all stores listed (or available) must ' + 'succeed. "True" by default meaning that we allow some ' + 'stores to fail and the status can be monitored from the ' + 'image metadata. If this is set to "False" the import will ' + 'be reverted should any of the uploads fail. Only usable ' + 'with "stores" or "all-stores".')) @utils.on_data_require_fields(DATA_FIELDS) def do_image_create_via_import(gc, args): """EXPERIMENTAL: Create a new image via image import. @@ -188,6 +210,10 @@ def do_image_create_via_import(gc, args): if args.import_method is None and (file_name or using_stdin): args.import_method = 'glance-direct' + if args.import_method == 'copy-image': + utils.exit("Import method 'copy-image' cannot be used " + "while creating the image.") + # determine whether the requested import method is valid import_methods = gc.images.get_import_info().get('import-methods') if args.import_method and args.import_method not in import_methods.get( @@ -198,9 +224,21 @@ def do_image_create_via_import(gc, args): # determine if backend is valid backend = None - if args.backend: - backend = args.backend + stores = getattr(args, "stores", None) + all_stores = getattr(args, "os_all_stores", None) + + if (args.store and (stores or all_stores)) or (stores and all_stores): + utils.exit("Only one of --store, --stores and --all-stores can be " + "provided") + elif args.store: + backend = args.store + # determine if backend is valid _validate_backend(backend, gc) + elif stores: + stores = str(stores).split(',') + for store in stores: + # determine if backend is valid + _validate_backend(store, gc) # make sure we have all and only correct inputs for the requested method if args.import_method is None: @@ -209,8 +247,16 @@ def do_image_create_via_import(gc, args): "method.") if args.import_method == 'glance-direct': if backend and not (file_name or using_stdin): - utils.exit("--backend option should only be provided with --file " + utils.exit("--store option should only be provided with --file " + "option or stdin for the glance-direct import method.") + if stores and not (file_name or using_stdin): + utils.exit("--stores option should only be provided with --file " "option or stdin for the glance-direct import method.") + if all_stores and not (file_name or using_stdin): + utils.exit("--all-stores option should only be provided with " + "--file option or stdin for the glance-direct import " + "method.") + if args.uri: utils.exit("You cannot specify a --uri with the glance-direct " "import method.") @@ -225,8 +271,14 @@ def do_image_create_via_import(gc, args): "for the glance-direct import method.") if args.import_method == 'web-download': if backend and not args.uri: - utils.exit("--backend option should only be provided with --uri " + utils.exit("--store option should only be provided with --uri " "option for the web-download import method.") + if stores and not args.uri: + utils.exit("--stores option should only be provided with --uri " + "option for the web-download import method.") + if all_stores and not args.uri: + utils.exit("--all-stores option should only be provided with " + "--uri option for the web-download import method.") if not args.uri: utils.exit("URI is required for web-download import method. " "Please use '--uri <uri>'.") @@ -246,6 +298,7 @@ def do_image_create_via_import(gc, args): args.size = None do_image_stage(gc, args) args.from_create = True + args.stores = stores do_image_import(gc, args) image = gc.images.get(args.id) finally: @@ -267,7 +320,7 @@ def _validate_backend(backend, gc): break if not valid_backend: - utils.exit("Backend '%s' is not valid for this cloud. Valid " + utils.exit("Store '%s' is not valid for this cloud. Valid " "values can be retrieved with stores-info command." % backend) @@ -327,6 +380,9 @@ def do_image_update(gc, args): action='append', dest='properties', default=[]) @utils.arg('--checksum', metavar='<CHECKSUM>', help=_('Displays images that match the MD5 checksum.')) +@utils.arg('--hash', dest='os_hash_value', default=None, + metavar='<HASH_VALUE>', + help=_('Displays images that match the specified hash value.')) @utils.arg('--tag', metavar='<TAG>', action='append', help=_("Filter images by a user-defined tag.")) @utils.arg('--sort-key', default=[], action='append', @@ -348,10 +404,17 @@ def do_image_update(gc, args): const=True, nargs='?', help="Filters results by hidden status. Default=None.") +@utils.arg('--include-stores', + metavar='[True|False]', + default=None, + type=strutils.bool_from_string, + const=True, + nargs='?', + help="Print backend store id.") def do_image_list(gc, args): """List images you can access.""" filter_keys = ['visibility', 'member_status', 'owner', 'checksum', 'tag', - 'os_hidden'] + 'os_hidden', 'os_hash_value'] filter_items = [(key, getattr(args, key)) for key in filter_keys] if args.properties: @@ -384,6 +447,9 @@ def do_image_list(gc, args): columns += ['Disk_format', 'Container_format', 'Size', 'Status', 'Owner'] + if args.include_stores: + columns += ['Stores'] + images = gc.images.list(**kwargs) utils.print_list(images, columns) @@ -490,6 +556,35 @@ def do_stores_info(gc, args): utils.print_dict(stores_info) +@utils.arg('id', metavar='<IMAGE_ID>', help=_('ID of image to update.')) +@utils.arg('--store', metavar='<STORE_ID>', required=True, + help=_('Store to delete image from.')) +def do_stores_delete(gc, args): + """Delete image from specific store.""" + try: + gc.images.delete_from_store(args.store, args.id) + except exc.HTTPNotFound: + utils.exit('Multi Backend support is not enabled or Image/store not ' + 'found.') + except (exc.HTTPForbidden, exc.HTTPException) as e: + msg = ("Unable to delete image '%s' from store '%s'. (%s)" % ( + args.id, + args.store, + e)) + utils.exit(msg) + + +@utils.arg('--allow-md5-fallback', action='store_true', + default=utils.env('OS_IMAGE_ALLOW_MD5_FALLBACK', default=False), + help=_('If os_hash_algo and os_hash_value properties are available ' + 'on the image, they will be used to validate the downloaded ' + 'image data. If the indicated secure hash algorithm is not ' + 'available on the client, the download will fail. Use this ' + 'flag to indicate that in such a case the legacy MD5 image ' + 'checksum should be used to validate the downloaded data. ' + 'You can also set the environment variable ' + 'OS_IMAGE_ALLOW_MD5_FALLBACK to any value to activate this ' + 'option.')) @utils.arg('--file', metavar='<FILE>', help=_('Local file to save downloaded image data to. ' 'If this is not specified and there is no redirection ' @@ -506,7 +601,8 @@ def do_image_download(gc, args): utils.exit(msg) try: - body = gc.images.data(args.id) + body = gc.images.data(args.id, + allow_md5_fallback=args.allow_md5_fallback) except (exc.HTTPForbidden, exc.HTTPException) as e: msg = "Unable to download image '%s'. (%s)" % (args.id, e) utils.exit(msg) @@ -534,14 +630,14 @@ def do_image_download(gc, args): help=_('Show upload progress bar.')) @utils.arg('id', metavar='<IMAGE_ID>', help=_('ID of image to upload data to.')) -@utils.arg('--backend', metavar='<STORE>', - default=utils.env('OS_IMAGE_BACKEND', default=None), +@utils.arg('--store', metavar='<STORE>', + default=utils.env('OS_IMAGE_STORE', default=None), help='Backend store to upload image to.') def do_image_upload(gc, args): """Upload data for a specific image.""" backend = None - if args.backend: - backend = args.backend + if args.store: + backend = args.store # determine if backend is valid _validate_backend(backend, gc) @@ -589,22 +685,59 @@ def do_image_stage(gc, args): help=_('URI to download the external image.')) @utils.arg('id', metavar='<IMAGE_ID>', help=_('ID of image to import.')) -@utils.arg('--backend', metavar='<STORE>', - default=utils.env('OS_IMAGE_BACKEND', default=None), +@utils.arg('--store', metavar='<STORE>', + default=utils.env('OS_IMAGE_STORE', default=None), help='Backend store to upload image to.') +@utils.arg('--stores', metavar='<STORES>', + default=utils.env('OS_IMAGE_STORES', default=None), + help='Stores to upload image to if multi-stores import available.') +@utils.arg('--all-stores', type=strutils.bool_from_string, + metavar='[True|False]', + default=None, + dest='os_all_stores', + help=_('"all-stores" can be ued instead of "stores"-list to ' + 'indicate that image should be imported all available ' + 'stores.')) +@utils.arg('--allow-failure', type=strutils.bool_from_string, + metavar='[True|False]', + dest='os_allow_failure', + default=utils.env('OS_IMAGE_ALLOW_FAILURE', default=True), + help=_('Indicator if all stores listed (or available) must ' + 'succeed. "True" by default meaning that we allow some ' + 'stores to fail and the status can be monitored from the ' + 'image metadata. If this is set to "False" the import will ' + 'be reverted should any of the uploads fail. Only usable ' + 'with "stores" or "all-stores".')) def do_image_import(gc, args): """Initiate the image import taskflow.""" - backend = None - if args.backend: - backend = args.backend + backend = getattr(args, "store", None) + stores = getattr(args, "stores", None) + all_stores = getattr(args, "os_all_stores", None) + allow_failure = getattr(args, "os_allow_failure", True) + + if not getattr(args, 'from_create', False): + if (args.store and (stores or all_stores)) or (stores and all_stores): + utils.exit("Only one of --store, --stores and --all-stores can be " + "provided") + elif args.store: + backend = args.store + # determine if backend is valid + _validate_backend(backend, gc) + elif stores: + stores = str(stores).split(',') + # determine if backend is valid - _validate_backend(backend, gc) + if stores: + for store in stores: + _validate_backend(store, gc) if getattr(args, 'from_create', False): # this command is being called "internally" so we can skip # validation -- just do the import and get out of here gc.images.image_import(args.id, args.import_method, args.uri, - backend=backend) + backend=backend, + stores=stores, all_stores=all_stores, + allow_failure=allow_failure) return # do input validation @@ -624,6 +757,10 @@ def do_image_import(gc, args): utils.exit("Import method should be 'web-download' if URI is " "provided.") + if args.import_method == 'copy-image' and not (stores or all_stores): + utils.exit("Provide either --stores or --all-stores for " + "'copy-image' import method.") + # check image properties image = gc.images.get(args.id) container_format = image.get('container_format') @@ -641,10 +778,16 @@ def do_image_import(gc, args): if image_status != 'queued': utils.exit("The 'web-download' import method can only be applied " "to an image in status 'queued'") + if args.import_method == 'copy-image': + if image_status != 'active': + utils.exit("The 'copy-image' import method can only be used on " + "an image with status 'active'.") # finally, do the import gc.images.image_import(args.id, args.import_method, args.uri, - backend=backend) + backend=backend, + stores=stores, all_stores=all_stores, + allow_failure=allow_failure) image = gc.images.get(args.id) utils.print_image(image) @@ -725,16 +868,30 @@ def do_image_tag_delete(gc, args): @utils.arg('--metadata', metavar='<STRING>', default='{}', help=_('Metadata associated with the location. ' 'Must be a valid JSON object (default: %(default)s)')) +@utils.arg('--checksum', metavar='<STRING>', + help=_('md5 checksum of image contents')) +@utils.arg('--hash-algo', metavar='<STRING>', + help=_('Multihash algorithm')) +@utils.arg('--hash-value', metavar='<STRING>', + help=_('Multihash value')) @utils.arg('id', metavar='<IMAGE_ID>', help=_('ID of image to which the location is to be added.')) def do_location_add(gc, args): """Add a location (and related metadata) to an image.""" + validation_data = {} + if args.checksum: + validation_data['checksum'] = args.checksum + if args.hash_algo: + validation_data['os_hash_algo'] = args.hash_algo + if args.hash_value: + validation_data['os_hash_value'] = args.hash_value try: metadata = json.loads(args.metadata) except ValueError: utils.exit('Metadata is not a valid JSON object.') else: - image = gc.images.add_location(args.id, args.url, metadata) + image = gc.images.add_location(args.id, args.url, metadata, + validation_data=validation_data) utils.print_dict(image) |