diff options
44 files changed, 945 insertions, 169 deletions
diff --git a/.zuul.yaml b/.zuul.yaml index 0a74da28d..04836e2fe 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,22 +1,4 @@ - job: - name: glance-code-constants-check - parent: tox - description: | - Tests to catch when code constants have gotten out of sync. - vars: - tox_envlist: gateonly - irrelevant-files: - - ^(test-|)requirements.txt$ - - ^.*\.rst$ - - ^api-ref/.*$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^tox.ini$ - - ^\.zuul\.yaml$ - -- job: name: glance-tox-oslo-tips-base parent: tox abstract: true @@ -311,7 +293,8 @@ templates: - check-requirements - integrated-gate-storage - - openstack-python3-zed-jobs + - openstack-python3-jobs + - openstack-python3-jobs-arm64 - periodic-stable-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 @@ -320,7 +303,6 @@ - openstack-tox-functional-py38-fips - openstack-tox-functional-py39 - glance-tox-functional-py39-rbac-defaults - - glance-code-constants-check - glance-ceph-thin-provisioning: voting: false irrelevant-files: &tempest-irrelevant-files diff --git a/api-ref/source/v2/samples/metadef-properties-list-response.json b/api-ref/source/v2/samples/metadef-properties-list-response.json index 7b34385aa..b373e8cdd 100644 --- a/api-ref/source/v2/samples/metadef-properties-list-response.json +++ b/api-ref/source/v2/samples/metadef-properties-list-response.json @@ -59,7 +59,7 @@ "type": "integer" }, "hw_vif_model": { - "description": "Specifies the model of virtual network interface device to use. The valid options depend on the configured hypervisor. KVM and QEMU: e1000, ne2k_pci, pcnet, rtl8139, virtio, e1000e and vmxnet3. VMware: e1000, e1000e, VirtualE1000, VirtualE1000e, VirtualPCNet32, VirtualSriovEthernetCard, and VirtualVmxnet. Xen: e1000, netfront, ne2k_pci, pcnet, and rtl8139.", + "description": "Specifies the model of virtual network interface device to use. The valid options depend on the configured hypervisor. KVM and QEMU: e1000, ne2k_pci, pcnet, rtl8139, virtio, e1000e and vmxnet3. VMware: e1000, e1000e, VirtualE1000, VirtualE1000e, VirtualPCNet32, and VirtualVmxnet. Xen: e1000, netfront, ne2k_pci, pcnet, and rtl8139.", "enum": [ "e1000", "ne2k_pci", @@ -71,7 +71,6 @@ "VirtualE1000", "VirtualE1000e", "VirtualPCNet32", - "VirtualSriovEthernetCard", "VirtualVmxnet", "netfront", "ne2k_pci" diff --git a/doc/source/admin/quotas.rst b/doc/source/admin/quotas.rst index a3894e3fc..5a79eeaa8 100644 --- a/doc/source/admin/quotas.rst +++ b/doc/source/admin/quotas.rst @@ -12,6 +12,11 @@ License for the specific language governing permissions and limitations under the License. +.. versionadded:: 23.0.0 (Xena) + + This functionality was first introduced in the 23.0.0 (Xena) release. Prior + to this, only global resource limits were supported. + .. _quotas: Per-Tenant Quotas diff --git a/doc/source/admin/useful-image-properties.rst b/doc/source/admin/useful-image-properties.rst index 0f36e9fdf..934e1f1a1 100644 --- a/doc/source/admin/useful-image-properties.rst +++ b/doc/source/admin/useful-image-properties.rst @@ -639,8 +639,7 @@ Here is a list of useful image properties and the values they expect. * ``KVM`` and ``QEMU``: ``e1000``, ``e1000e``, ``ne2k_pci``, ``pcnet``, ``rtl8139``, ``virtio`` and vmxnet3``. * VMware: ``e1000``, ``e1000e``, ``VirtualE1000``, ``VirtualE1000e``, - ``VirtualPCNet32``, ``VirtualSriovEthernetCard``, ``VirtualVmxnet`` - and ``VirtualVmxnet3``. + ``VirtualPCNet32``, ``VirtualVmxnet`` and ``VirtualVmxnet3``. * Xen: ``e1000``, ``netfront``, ``ne2k_pci``, ``pcnet``, and ``rtl8139``. diff --git a/doc/source/cli/glancemanage.rst b/doc/source/cli/glancemanage.rst index 628de96a5..03d47f931 100644 --- a/doc/source/cli/glancemanage.rst +++ b/doc/source/cli/glancemanage.rst @@ -82,6 +82,38 @@ COMMANDS metadef_objects, metadef_resource_types, metadef_namespaces and metadef_properties. +``db_purge`` + Purge deleted rows older than a given age from glance db tables. + + This command interprets the following options when it is invoked: + + --max_rows Purge deleted rows older than age in days (default + value if not specified: 100) + --age_in_days Limit number of records to delete (default value if + not specified: 30 days) + + WARNING: This function is useful primarily in test systems. We do not + recommend its use in production systems unless you have reviewed + OpenStack Security Note `OSSN-0075`_ and understand the risk involved. + + .. _`OSSN-0075`: https://wiki.openstack.org/wiki/OSSN/OSSN-0075 + +``db_purge_images_table`` + Purge deleted rows older than a given age from images db tables. + + This command interprets the following options when it is invoked: + + --max_rows Purge deleted rows older than age in days (default + value if not specified: 100) + --age_in_days Limit number of records to delete (default value if + not specified: 30 days) + + WARNING: This function is useful primarily in test systems. We do not + recommend its use in production systems unless you have reviewed + OpenStack Security Note `OSSN-0075`_ and understand the risk involved. + + .. _`OSSN-0075`: https://wiki.openstack.org/wiki/OSSN/OSSN-0075 + OPTIONS ======= diff --git a/doc/source/configuration/configuring.rst b/doc/source/configuration/configuring.rst index 5298fcd3b..80917d423 100644 --- a/doc/source/configuration/configuring.rst +++ b/doc/source/configuration/configuring.rst @@ -735,9 +735,8 @@ Debian-based distributions. `This option is specific to the RBD storage backend.` Sets the RADOS user to authenticate as. This is only needed - when `RADOS authentication <https://docs.ceph.com/docs/emperor/rados/operations/authentication/>`_ - is `enabled. - <https://docs.ceph.com/docs/emperor/rados/operations/authentication/#enabling-cephx>`_ + when `RADOS authentication <https://docs.ceph.com/en/latest/rados/configuration/auth-config-ref/>`_ + is enabled. A keyring must be set for this user in the Ceph configuration file, e.g. with a user ``glance``:: diff --git a/doc/source/install/configure-quotas.rst b/doc/source/install/configure-quotas.rst index 075b7b0ae..c3a1d3193 100644 --- a/doc/source/install/configure-quotas.rst +++ b/doc/source/install/configure-quotas.rst @@ -6,20 +6,23 @@ auth_url = http://controller:5000 auth_type = password user_domain_id = default - username = MY_SERVICE + username = glance system_scope = all - password = MY_PASSWORD - endpoint_id = ENDPOINT_ID + password = GLANCE_PASS + endpoint_id = 340be3625e9b4239a6415d034e98aace region_name = RegionOne .. end - Make sure that the MY_SERVICE account has reader access to + Replace ``GLANCE_PASS`` with the password you chose for the + ``glance`` user in the Identity service. + + Make sure that the glance account has reader access to system-scope resources (like limits): .. code-block:: console - $ openstack role add --user MY_SERVICE --user-domain Default --system all reader + $ openstack role add --user glance --user-domain Default --system all reader .. end @@ -33,7 +36,7 @@ .. code-block:: ini [DEFAULT] - use_keystone_quotas = True + use_keystone_limits = True .. end diff --git a/doc/source/install/register-quotas.rst b/doc/source/install/register-quotas.rst index 650ce15da..260c04fe5 100644 --- a/doc/source/install/register-quotas.rst +++ b/doc/source/install/register-quotas.rst @@ -63,4 +63,4 @@ .. end - Be sure to also set ``use_keystone_quotas=True`` in your ``glance-api.conf`` file. + Be sure to also set ``use_keystone_limits=True`` in your ``glance-api.conf`` file. diff --git a/etc/glance-api.conf b/etc/glance-api.conf index 9db3f17ec..785ddc96b 100644 --- a/etc/glance-api.conf +++ b/etc/glance-api.conf @@ -1771,8 +1771,12 @@ # (string value) #mysql_sql_mode = TRADITIONAL -# If True, transparently enables support for handling MySQL Cluster (NDB). -# (boolean value) +# DEPRECATED: If True, transparently enables support for handling MySQL Cluster +# (NDB). (boolean value) +# This option is deprecated for removal since 12.1.0. +# Its value may be silently ignored in the future. +# Reason: Support for the MySQL NDB Cluster storage engine has been deprecated +# and will be removed in a future release. #mysql_enable_ndb = false # Connections which have been present in the connection pool longer than this @@ -5619,8 +5623,9 @@ # default. If this option is equal to False then the health check heartbeat will # inherit the execution model from the parent process. For example if the parent # process has monkey patched the stdlib by using eventlet/greenlet then the -# heartbeat will be run through a green thread. (boolean value) -#heartbeat_in_pthread = true +# heartbeat will be run through a green thread. This option should be set to +# True only for the wsgi services. (boolean value) +#heartbeat_in_pthread = false # How long to wait before reconnecting in response to an AMQP consumer cancel # notification. (floating point value) diff --git a/etc/glance-manage.conf b/etc/glance-manage.conf index f2dcce39f..55eebdca5 100644 --- a/etc/glance-manage.conf +++ b/etc/glance-manage.conf @@ -178,8 +178,12 @@ # (string value) #mysql_sql_mode = TRADITIONAL -# If True, transparently enables support for handling MySQL Cluster (NDB). -# (boolean value) +# DEPRECATED: If True, transparently enables support for handling MySQL Cluster +# (NDB). (boolean value) +# This option is deprecated for removal since 12.1.0. +# Its value may be silently ignored in the future. +# Reason: Support for the MySQL NDB Cluster storage engine has been deprecated +# and will be removed in a future release. #mysql_enable_ndb = false # Connections which have been present in the connection pool longer than this diff --git a/etc/glance-scrubber.conf b/etc/glance-scrubber.conf index 02a0e3211..a71cb2907 100644 --- a/etc/glance-scrubber.conf +++ b/etc/glance-scrubber.conf @@ -789,8 +789,12 @@ # (string value) #mysql_sql_mode = TRADITIONAL -# If True, transparently enables support for handling MySQL Cluster (NDB). -# (boolean value) +# DEPRECATED: If True, transparently enables support for handling MySQL Cluster +# (NDB). (boolean value) +# This option is deprecated for removal since 12.1.0. +# Its value may be silently ignored in the future. +# Reason: Support for the MySQL NDB Cluster storage engine has been deprecated +# and will be removed in a future release. #mysql_enable_ndb = false # Connections which have been present in the connection pool longer than this diff --git a/etc/metadefs/compute-vmware.json b/etc/metadefs/compute-vmware.json index 460bec825..7ea8c3287 100644 --- a/etc/metadefs/compute-vmware.json +++ b/etc/metadefs/compute-vmware.json @@ -193,7 +193,7 @@ }, "hw_vif_model": { "title": "Virtual Network Interface", - "description": "Specifies the model of virtual network interface device to use. The valid options depend on the hypervisor. VMware driver supported options: e1000, e1000e, VirtualE1000, VirtualE1000e, VirtualPCNet32, VirtualSriovEthernetCard, and VirtualVmxnet.", + "description": "Specifies the model of virtual network interface device to use. The valid options depend on the hypervisor. VMware driver supported options: e1000, e1000e, VirtualE1000, VirtualE1000e, VirtualPCNet32, and VirtualVmxnet.", "type": "string", "enum": [ "e1000", @@ -201,7 +201,6 @@ "VirtualE1000", "VirtualE1000e", "VirtualPCNet32", - "VirtualSriovEthernetCard", "VirtualVmxnet", "VirtualVmxnet3" ], diff --git a/glance/async_/flows/plugins/image_conversion.py b/glance/async_/flows/plugins/image_conversion.py index 32c7b7fe0..e977764fa 100644 --- a/glance/async_/flows/plugins/image_conversion.py +++ b/glance/async_/flows/plugins/image_conversion.py @@ -116,6 +116,29 @@ class _ConvertImage(task.Task): virtual_size = metadata.get('virtual-size', 0) action.set_image_attribute(virtual_size=virtual_size) + if 'backing-filename' in metadata: + LOG.warning('Refusing to process QCOW image with a backing file') + raise RuntimeError( + 'QCOW images with backing files are not allowed') + + if metadata.get('format') == 'vmdk': + create_type = metadata.get( + 'format-specific', {}).get( + 'data', {}).get('create-type') + allowed = CONF.image_format.vmdk_allowed_types + if not create_type: + raise RuntimeError(_('Unable to determine VMDK create-type')) + if not len(allowed): + LOG.warning(_('Refusing to process VMDK file as ' + 'vmdk_allowed_types is empty')) + raise RuntimeError(_('Image is a VMDK, but no VMDK createType ' + 'is specified')) + if create_type not in allowed: + LOG.warning(_('Refusing to process VMDK file with create-type ' + 'of %r which is not in allowed set of: %s'), + create_type, ','.join(allowed)) + raise RuntimeError(_('Invalid VMDK create-type specified')) + if source_format == target_format: LOG.debug("Source is already in target format, " "not doing conversion for %s", self.image_id) diff --git a/glance/common/config.py b/glance/common/config.py index dd7e1b6e9..7891daccf 100644 --- a/glance/common/config.py +++ b/glance/common/config.py @@ -99,6 +99,18 @@ image_format_opts = [ "image attribute"), deprecated_opts=[cfg.DeprecatedOpt('disk_formats', group='DEFAULT')]), + cfg.ListOpt('vmdk_allowed_types', + default=['streamOptimized', 'monolithicSparse'], + help=_("A list of strings describing allowed VMDK " + "'create-type' subformats that will be allowed. " + "This is recommended to only include " + "single-file-with-sparse-header variants to avoid " + "potential host file exposure due to processing named " + "extents. If this list is empty, then no VDMK image " + "types allowed. Note that this is currently only " + "checked during image conversion (if enabled), and " + "limits the types of VMDK images we will convert " + "from.")), ] task_opts = [ cfg.IntOpt('task_time_to_live', diff --git a/glance/common/format_inspector.py b/glance/common/format_inspector.py index fbd20273b..550cceadb 100755 --- a/glance/common/format_inspector.py +++ b/glance/common/format_inspector.py @@ -345,6 +345,7 @@ class VHDXInspector(FileInspector): """ METAREGION = '8B7CA206-4790-4B9A-B8FE-575F050F886E' VIRTUAL_DISK_SIZE = '2FA54224-CD1B-4876-B211-5DBED83BF4B8' + VHDX_METADATA_TABLE_MAX_SIZE = 32 * 2048 # From qemu def __init__(self, *a, **k): super(VHDXInspector, self).__init__(*a, **k) @@ -459,6 +460,8 @@ class VHDXInspector(FileInspector): item_offset, item_length, _reserved = struct.unpack( '<III', meta_buffer[entry_offset + 16:entry_offset + 28]) + item_length = min(item_length, + self.VHDX_METADATA_TABLE_MAX_SIZE) self.region('metadata').length = len(meta_buffer) self._log.debug('Found entry at offset %x', item_offset) # Metadata item offset is from the beginning of the metadata @@ -516,6 +519,12 @@ class VMDKInspector(FileInspector): variable number of 512 byte sectors, but is just text defining the layout of the disk. """ + + # The beginning and max size of the descriptor is also hardcoded in Qemu + # at 0x200 and 1MB - 1 + DESC_OFFSET = 0x200 + DESC_MAX_SIZE = (1 << 20) - 1 + def __init__(self, *a, **k): super(VMDKInspector, self).__init__(*a, **k) self.new_region('header', CaptureRegion(0, 512)) @@ -532,15 +541,22 @@ class VMDKInspector(FileInspector): if sig != b'KDMV': raise ImageFormatError('Signature KDMV not found: %r' % sig) - return if ver not in (1, 2, 3): raise ImageFormatError('Unsupported format version %i' % ver) - return + + # Since we parse both desc_sec and desc_num (the location of the + # VMDK's descriptor, expressed in 512 bytes sectors) we enforce a + # check on the bounds to create a reasonable CaptureRegion. This + # is similar to how it's done in qemu. + desc_offset = desc_sec * 512 + desc_size = min(desc_num * 512, self.DESC_MAX_SIZE) + if desc_offset != self.DESC_OFFSET: + raise ImageFormatError("Wrong descriptor location") if not self.has_region('descriptor'): self.new_region('descriptor', CaptureRegion( - desc_sec * 512, desc_num * 512)) + desc_offset, desc_size)) @property def format_match(self): @@ -567,7 +583,7 @@ class VMDKInspector(FileInspector): else: vmdktype = b'formatnotfound' if vmdktype != b'monolithicSparse': - raise ImageFormatError('Unsupported VMDK format %s' % vmdktype) + LOG.warning('Unsupported VMDK format %s', vmdktype) return 0 # If we have the descriptor, we definitely have the header diff --git a/glance/db/migration.py b/glance/db/migration.py index f6fdd8d08..1977de469 100644 --- a/glance/db/migration.py +++ b/glance/db/migration.py @@ -29,5 +29,5 @@ db_options.set_defaults(cfg.CONF) # Migration-related constants EXPAND_BRANCH = 'expand' CONTRACT_BRANCH = 'contract' -CURRENT_RELEASE = 'zed' +CURRENT_RELEASE = '2023_1' ALEMBIC_INIT_VERSION = 'liberty' diff --git a/glance/db/sqlalchemy/alembic_migrations/data_migrations/2023_1_migrate01_empty.py b/glance/db/sqlalchemy/alembic_migrations/data_migrations/2023_1_migrate01_empty.py new file mode 100644 index 000000000..893d66db9 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/data_migrations/2023_1_migrate01_empty.py @@ -0,0 +1,26 @@ +# Copyright (C) 2021 RedHat Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def has_migrations(engine): + """Returns true if at least one data row can be migrated.""" + + return False + + +def migrate(engine): + """Return the number of rows migrated.""" + + return 0 diff --git a/glance/db/sqlalchemy/alembic_migrations/data_migrations/xena_migrate01_empty.py b/glance/db/sqlalchemy/alembic_migrations/data_migrations/xena_migrate01_empty.py new file mode 100644 index 000000000..893d66db9 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/data_migrations/xena_migrate01_empty.py @@ -0,0 +1,26 @@ +# Copyright (C) 2021 RedHat Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def has_migrations(engine): + """Returns true if at least one data row can be migrated.""" + + return False + + +def migrate(engine): + """Return the number of rows migrated.""" + + return 0 diff --git a/glance/db/sqlalchemy/alembic_migrations/data_migrations/yoga_migrate01_empty.py b/glance/db/sqlalchemy/alembic_migrations/data_migrations/yoga_migrate01_empty.py new file mode 100644 index 000000000..893d66db9 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/data_migrations/yoga_migrate01_empty.py @@ -0,0 +1,26 @@ +# Copyright (C) 2021 RedHat Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def has_migrations(engine): + """Returns true if at least one data row can be migrated.""" + + return False + + +def migrate(engine): + """Return the number of rows migrated.""" + + return 0 diff --git a/glance/db/sqlalchemy/alembic_migrations/data_migrations/zed_migrate01_empty.py b/glance/db/sqlalchemy/alembic_migrations/data_migrations/zed_migrate01_empty.py new file mode 100644 index 000000000..893d66db9 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/data_migrations/zed_migrate01_empty.py @@ -0,0 +1,26 @@ +# Copyright (C) 2021 RedHat Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def has_migrations(engine): + """Returns true if at least one data row can be migrated.""" + + return False + + +def migrate(engine): + """Return the number of rows migrated.""" + + return 0 diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/2023_1_contract01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/2023_1_contract01_empty.py new file mode 100644 index 000000000..b9afc5e76 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/2023_1_contract01_empty.py @@ -0,0 +1,25 @@ +# Copyright (C) 2020 RedHat Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# revision identifiers, used by Alembic. +revision = '2023_1_contract01' +down_revision = 'zed_contract01' +branch_labels = None +depends_on = '2023_1_expand01' + + +def upgrade(): + pass diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/2023_1_expand01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/2023_1_expand01_empty.py new file mode 100644 index 000000000..334766c2f --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/2023_1_expand01_empty.py @@ -0,0 +1,30 @@ +# Copyright (C) 2020 RedHat Inc +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""empty expand for symmetry with 2023_1_expand01 + +Revision ID: ussuri_expand01 +Revises: train_expand01 +Create Date: 2020-01-03 11:55:16.657499 + +""" + +# revision identifiers, used by Alembic. +revision = '2023_1_expand01' +down_revision = 'zed_expand01' +branch_labels = None +depends_on = None + + +def upgrade(): + pass diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/xena_contract01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/xena_contract01_empty.py new file mode 100644 index 000000000..ecbb75cfd --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/xena_contract01_empty.py @@ -0,0 +1,25 @@ +# Copyright (C) 2020 RedHat Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# revision identifiers, used by Alembic. +revision = 'xena_contract01' +down_revision = 'wallaby_contract01' +branch_labels = None +depends_on = 'xena_expand01' + + +def upgrade(): + pass diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/xena_expand01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/xena_expand01_empty.py new file mode 100644 index 000000000..42306928d --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/xena_expand01_empty.py @@ -0,0 +1,30 @@ +# Copyright (C) 2020 RedHat Inc +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""empty expand for symmetry with 2023_1_expand01 + +Revision ID: ussuri_expand01 +Revises: train_expand01 +Create Date: 2020-01-03 11:55:16.657499 + +""" + +# revision identifiers, used by Alembic. +revision = 'xena_expand01' +down_revision = 'wallaby_expand01' +branch_labels = None +depends_on = None + + +def upgrade(): + pass diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/yoga_contract01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/yoga_contract01_empty.py new file mode 100644 index 000000000..082be9994 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/yoga_contract01_empty.py @@ -0,0 +1,25 @@ +# Copyright (C) 2020 RedHat Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# revision identifiers, used by Alembic. +revision = 'yoga_contract01' +down_revision = 'xena_contract01' +branch_labels = None +depends_on = 'yoga_expand01' + + +def upgrade(): + pass diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/yoga_expand01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/yoga_expand01_empty.py new file mode 100644 index 000000000..f984a2fa5 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/yoga_expand01_empty.py @@ -0,0 +1,30 @@ +# Copyright (C) 2020 RedHat Inc +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""empty expand for symmetry with 2023_1_expand01 + +Revision ID: ussuri_expand01 +Revises: train_expand01 +Create Date: 2020-01-03 11:55:16.657499 + +""" + +# revision identifiers, used by Alembic. +revision = 'yoga_expand01' +down_revision = 'xena_expand01' +branch_labels = None +depends_on = None + + +def upgrade(): + pass diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/zed_contract01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/zed_contract01_empty.py new file mode 100644 index 000000000..32d36ddfb --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/zed_contract01_empty.py @@ -0,0 +1,25 @@ +# Copyright (C) 2020 RedHat Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# revision identifiers, used by Alembic. +revision = 'zed_contract01' +down_revision = 'yoga_contract01' +branch_labels = None +depends_on = 'zed_expand01' + + +def upgrade(): + pass diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/zed_expand01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/zed_expand01_empty.py new file mode 100644 index 000000000..14e6e081d --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/zed_expand01_empty.py @@ -0,0 +1,30 @@ +# Copyright (C) 2020 RedHat Inc +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""empty expand for symmetry with 2023_1_expand01 + +Revision ID: ussuri_expand01 +Revises: train_expand01 +Create Date: 2020-01-03 11:55:16.657499 + +""" + +# revision identifiers, used by Alembic. +revision = 'zed_expand01' +down_revision = 'yoga_expand01' +branch_labels = None +depends_on = None + + +def upgrade(): + pass diff --git a/glance/locale/en_GB/LC_MESSAGES/glance.po b/glance/locale/en_GB/LC_MESSAGES/glance.po index 9b92a6b95..405d4fd30 100644 --- a/glance/locale/en_GB/LC_MESSAGES/glance.po +++ b/glance/locale/en_GB/LC_MESSAGES/glance.po @@ -15,11 +15,11 @@ msgid "" msgstr "" "Project-Id-Version: glance VERSION\n" "Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n" -"POT-Creation-Date: 2022-08-24 12:13+0000\n" +"POT-Creation-Date: 2022-09-01 19:25+0000\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"PO-Revision-Date: 2022-08-02 08:42+0000\n" +"PO-Revision-Date: 2022-09-05 10:23+0000\n" "Last-Translator: Andi Chandler <andi@gowling.com>\n" "Language: en_GB\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" @@ -37,6 +37,7 @@ msgid "" "\n" " 'glance-direct', 'copy-image' and 'web-download' are enabled by " "default.\n" +" 'glance-download' is available, but requires federated deployments.\n" "\n" " Related options:\n" " * [DEFAULT]/node_staging_uri" @@ -46,6 +47,7 @@ msgstr "" "\n" " 'glance-direct', 'copy-image' and 'web-download' are enabled by " "default.\n" +" 'glance-download' is available, but requires federated deployments.\n" "\n" " Related options:\n" " * [DEFAULT]/node_staging_uri" @@ -1954,6 +1956,37 @@ msgstr "" msgid "" "\n" +"Specify metadata prefix to be set on the target image when using\n" +"glance-download. All other properties coming from the source image won't be " +"set\n" +"on the target image. If specified metadata does not exist on the source " +"image\n" +"it won't be set on the target image. Note you can't set the os_glance " +"prefix\n" +"as it is reserved by glance, so the related properties won't be set on the\n" +"target image.\n" +"\n" +"Possible values:\n" +" * List containing extra_properties prefixes: ['os_', 'architecture']\n" +"\n" +msgstr "" +"\n" +"Specify metadata prefix to be set on the target image when using\n" +"glance-download. All other properties coming from the source image won't be " +"set\n" +"on the target image. If specified metadata does not exist on the source " +"image\n" +"it won't be set on the target image. Note you can't set the os_glance " +"prefix\n" +"as it is reserved by Glance, so the related properties won't be set on the\n" +"target image.\n" +"\n" +"Possible values:\n" +" * List containing extra_properties prefixes: ['os_', 'architecture']\n" +"\n" + +msgid "" +"\n" "Specify name of user roles to be ignored for injecting metadata\n" "properties in the image.\n" "\n" @@ -2790,6 +2823,10 @@ msgid "%(cls)s exception was raised in the last rpc call: %(val)s" msgstr "%(cls)s exception was raised in the last rpc call: %(val)s" #, python-format +msgid "%(interface)s glance endpoint not found for region %(region)s" +msgstr "%(interface)s glance endpoint not found for region %(region)s" + +#, python-format msgid "%(m_id)s not found in the member list of the image %(i_id)s." msgstr "%(m_id)s not found in the member list of the image %(i_id)s." @@ -2896,6 +2933,12 @@ msgstr "'disk_format' needs to be set before import" msgid "'glance-direct' method is not available at this site." msgstr "'glance-direct' method is not available at this site." +msgid "'glance_image_id' needs to be set for glance-download import method" +msgstr "'glance_image_id' needs to be set for glance-download import method" + +msgid "'glance_region' needs to be set for glance-download import method" +msgstr "'glance_region' needs to be set for glance-download import method" + msgid "'node_staging_uri' is not set correctly. Could not load staging store." msgstr "'node_staging_uri' is not set correctly. Could not load staging store." @@ -3565,6 +3608,10 @@ msgstr "Image is already present at store '%s'" msgid "Image member limit exceeded for image %(id)s: %(e)s:" msgstr "Image member limit exceeded for image %(id)s: %(e)s:" +#, python-format +msgid "Image needs to be in 'queued' state to use '%s' method" +msgstr "Image needs to be in 'queued' state to use '%s' method" + msgid "Image needs to be staged before 'glance-direct' method can be used" msgstr "Image needs to be staged before 'glance-direct' method can be used" @@ -4163,6 +4210,10 @@ msgid "Reloading %(serv)s (pid %(pid)s) with signal(%(sig)s)" msgstr "Reloading %(serv)s (pid %(pid)s) with signal(%(sig)s)" #, python-format +msgid "Remote image id does not look like a UUID: %s" +msgstr "Remote image id does not look like a UUID: %s" + +#, python-format msgid "Removing stale pid file %s" msgstr "Removing stale pid file %s" @@ -4204,6 +4255,10 @@ msgstr "Server worker creation failed: %(reason)s." msgid "Signature verification failed" msgstr "Signature verification failed" +#, python-format +msgid "Size attribute of remote image %s could not be determined." +msgstr "Size attribute of remote image %s could not be determined." + msgid "Size of image file in bytes" msgstr "Size of image file in bytes" @@ -4224,6 +4279,10 @@ msgstr "Sort direction supplied was not valid." msgid "Sort key supplied was not valid." msgstr "Sort key supplied was not valid." +#, python-format +msgid "Source image status should be active instead of %s" +msgstr "Source image status should be active instead of %s" + msgid "" "Specifies the prefix to use for the given resource type. Any properties in " "the namespace should be prefixed with this prefix when being applied to the " diff --git a/glance/tests/functional/db/test_migrations.py b/glance/tests/functional/db/test_migrations.py index 0f7f0b4df..150af726d 100644 --- a/glance/tests/functional/db/test_migrations.py +++ b/glance/tests/functional/db/test_migrations.py @@ -63,21 +63,44 @@ class TestVersions(test_utils.BaseTestCase): for prefix in exception_releases]): continue - # File format should be release_phaseNN_description.py - try: - _rest = '' # noqa - release, phasever, _rest = version_file.split('_', 2) - except ValueError: - release = phasever = '' - phase = ''.join(x for x in phasever if x.isalpha()) - # Grab the non-numeric part of phaseNN - if phase not in required_phases: - # Help make sure that going forward developers stick to the - # consistent format. - self.fail('Migration files should be in the form of: ' - 'release_phaseNN_some_description.py ' - '(while processing %r)' % version_file) - releases[release].add(phase) + # For legacy database scripts does not starts with + # YYYY i.e. pre Antelope + if not version_file.split('_', 2)[0].isnumeric(): + # File format should be release_phaseNN_description.py + try: + _rest = '' # noqa + release, phasever, _rest = version_file.split('_', 2) + except ValueError: + release = phasever = '' + phase = ''.join(x for x in phasever if x.isalpha()) + # Grab the non-numeric part of phaseNN + if phase not in required_phases: + # Help make sure that going forward developers stick to the + # consistent format. + self.fail('Migration files should be in the form of: ' + 'release_phaseNN_some_description.py ' + '(while processing %r)' % version_file) + releases[release].add(phase) + else: + # For new database scripts i.e. Antelope onwards + # File format should be + # releaseYear_releaseN_phaseNN_description.py + # For example 2023_1_expand01_empty.py + try: + _rest = '' # noqa + release_y, release_n, phasever, _rest = version_file.split( + '_', 3) + except ValueError: + release_y = phasever = '' + phase = ''.join(x for x in phasever if x.isalpha()) + # Grab the non-numeric part of phaseNN + if phase not in required_phases: + # Help make sure that going forward developers stick to the + # consistent format. + self.fail('Migration files should be in the form of: ' + 'releaseYear_releaseN_phaseNN_description.py ' + '(while processing %r)' % version_file) + releases[release_y].add(phase) for release, phases in releases.items(): missing = required_phases - phases diff --git a/glance/tests/functional/v2/test_legacy_update_cinder_store.py b/glance/tests/functional/v2/test_legacy_update_cinder_store.py index 3911711ab..d42ae12c1 100644 --- a/glance/tests/functional/v2/test_legacy_update_cinder_store.py +++ b/glance/tests/functional/v2/test_legacy_update_cinder_store.py @@ -19,7 +19,6 @@ import uuid from cinderclient.v3 import client as cinderclient import glance_store -from glance_store._drivers import cinder from oslo_config import cfg from oslo_log import log as logging from oslo_utils import strutils @@ -27,6 +26,16 @@ from oslo_utils import strutils from glance.common import wsgi from glance.tests import functional +# Keeping backward compatibility to support importing from old +# path +try: + from glance_store._drivers.cinder import base + from glance_store._drivers.cinder import store as cinder +except ImportError: + from glance_store._drivers import cinder + base = mock.Mock() + + LOG = logging.getLogger(__name__) CONF = cfg.CONF @@ -135,6 +144,7 @@ class TestLegacyUpdateCinderStore(functional.SynchronousAPIBase): volume.status = status_expected return volume + @mock.patch.object(base, 'connector') @mock.patch.object(cinderclient, 'Client') @mock.patch.object(cinder.Store, 'temporary_chown') @mock.patch.object(cinder, 'connector') @@ -143,7 +153,8 @@ class TestLegacyUpdateCinderStore(functional.SynchronousAPIBase): @mock.patch.object(strutils, 'mask_dict_password') @mock.patch.object(socket, 'getaddrinfo') def test_create_image(self, mock_host_addr, mock_mask_pass, mock_wait, - mock_open, mock_connector, mock_chown, mocked_cc): + mock_open, mock_connector, mock_chown, mocked_cc, + mock_base): # setup multiple cinder stores self.setup_multiple_stores() self.start_server() @@ -165,6 +176,7 @@ class TestLegacyUpdateCinderStore(functional.SynchronousAPIBase): mock_chown.assert_called() mock_connector.get_connector_properties.assert_called() + @mock.patch.object(base, 'connector') @mock.patch.object(cinderclient, 'Client') @mock.patch.object(cinder.Store, 'temporary_chown') @mock.patch.object(cinder, 'connector') @@ -174,7 +186,7 @@ class TestLegacyUpdateCinderStore(functional.SynchronousAPIBase): @mock.patch.object(socket, 'getaddrinfo') def test_migrate_image_after_upgrade(self, mock_host_addr, mock_mask_pass, mock_wait, mock_open, mock_connector, - mock_chown, mocked_cc): + mock_chown, mocked_cc, mock_base): """Test to check if an image is successfully migrated when we upgrade from a single cinder store to multiple cinder stores. @@ -213,6 +225,7 @@ class TestLegacyUpdateCinderStore(functional.SynchronousAPIBase): mock_chown.assert_called() mock_connector.get_connector_properties.assert_called() + @mock.patch.object(base, 'connector') @mock.patch.object(cinderclient, 'Client') @mock.patch.object(cinder.Store, 'temporary_chown') @mock.patch.object(cinder, 'connector') @@ -224,7 +237,8 @@ class TestLegacyUpdateCinderStore(functional.SynchronousAPIBase): mock_mask_pass, mock_wait, mock_open, mock_connector, - mock_chown, mocked_cc): + mock_chown, mocked_cc, + mock_base): """Test to check if an image is successfully migrated when we upgrade from a single cinder store to multiple cinder stores, and that GETs from non-owners in the meantime are not interrupted. diff --git a/glance/tests/gate/README b/glance/tests/gate/README deleted file mode 100644 index 05636e9ac..000000000 --- a/glance/tests/gate/README +++ /dev/null @@ -1,11 +0,0 @@ -=============== -Gate-only tests -=============== - -These tests catch configuration problems for some code constants that -must be maintained manually. We have them separated out from the other -tests so that they can easily be run in their own gate job and don't -affect local development. - -It would be nice if someone with some free time could figure out how -to make these changes automatic (or unnecessary) ... diff --git a/glance/tests/gate/__init__.py b/glance/tests/gate/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/glance/tests/gate/__init__.py +++ /dev/null diff --git a/glance/tests/gate/test_data_migration_version.py b/glance/tests/gate/test_data_migration_version.py deleted file mode 100644 index 992e89001..000000000 --- a/glance/tests/gate/test_data_migration_version.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2019 Red Hat, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import testtools - -from glance.db.migration import CURRENT_RELEASE -from glance.version import version_info - - -class TestDataMigrationVersion(testtools.TestCase): - - def test_migration_version(self): - """Make sure the data migration version info has been updated.""" - - release_number = int(version_info.version_string().split('.', 1)[0]) - - # by rule, release names must be composed of the 26 letters of the - # ISO Latin alphabet (ord('A')==65, ord('Z')==90) - release_letter = str(CURRENT_RELEASE[:1].upper()).encode('ascii') - - # Convert release letter into an int in [1:26]. The first - # glance release was 'Bexar'. - converted_release_letter = (ord(release_letter) - - ord(u'B'.encode('ascii')) + 1) - - # Project the release number into [1:26] - converted_release_number = release_number % 26 - - # Prepare for the worst with a super-informative message - msg = ('\n\n' - 'EMERGENCY!\n' - 'glance.db.migration.CURRENT_RELEASE is out of sync ' - 'with the glance version.\n' - ' CURRENT_RELEASE: %s\n' - ' glance version: %s\n' - 'glance.db.migration.CURRENT_RELEASE needs to be ' - 'updated IMMEDIATELY.\n' - 'The gate will be wedged until the update is made.\n' - 'EMERGENCY!\n' - '\n') % (CURRENT_RELEASE, - version_info.version_string()) - - self.assertEqual(converted_release_letter, - converted_release_number, - msg) diff --git a/glance/tests/unit/async_/flows/plugins/test_image_conversion.py b/glance/tests/unit/async_/flows/plugins/test_image_conversion.py index 77d68acf8..a60e2e1a5 100644 --- a/glance/tests/unit/async_/flows/plugins/test_image_conversion.py +++ b/glance/tests/unit/async_/flows/plugins/test_image_conversion.py @@ -172,6 +172,53 @@ class TestConvertImageTask(test_utils.BaseTestCase): # Make sure we did not update the image self.img_repo.save.assert_not_called() + def test_image_convert_invalid_qcow(self): + data = {'format': 'qcow2', + 'backing-filename': '/etc/hosts'} + + convert = self._setup_image_convert_info_fail() + with mock.patch.object(processutils, 'execute') as exc_mock: + exc_mock.return_value = json.dumps(data), '' + e = self.assertRaises(RuntimeError, + convert.execute, 'file:///test/path.qcow') + self.assertEqual('QCOW images with backing files are not allowed', + str(e)) + + def _test_image_convert_invalid_vmdk(self): + data = {'format': 'vmdk', + 'format-specific': { + 'data': { + 'create-type': 'monolithicFlat', + }}} + + convert = self._setup_image_convert_info_fail() + with mock.patch.object(processutils, 'execute') as exc_mock: + exc_mock.return_value = json.dumps(data), '' + convert.execute('file:///test/path.vmdk') + + def test_image_convert_invalid_vmdk(self): + e = self.assertRaises(RuntimeError, + self._test_image_convert_invalid_vmdk) + self.assertEqual('Invalid VMDK create-type specified', str(e)) + + def test_image_convert_valid_vmdk_no_types(self): + with mock.patch.object(CONF.image_format, 'vmdk_allowed_types', + new=[]): + # We make it past the VMDK check and fail because our file + # does not exist + e = self.assertRaises(RuntimeError, + self._test_image_convert_invalid_vmdk) + self.assertEqual('Image is a VMDK, but no VMDK createType is ' + 'specified', str(e)) + + def test_image_convert_valid_vmdk(self): + with mock.patch.object(CONF.image_format, 'vmdk_allowed_types', + new=['monolithicSparse', 'monolithicFlat']): + # We make it past the VMDK check and fail because our file + # does not exist + self.assertRaises(FileNotFoundError, + self._test_image_convert_invalid_vmdk) + def test_image_convert_fails(self): convert = self._setup_image_convert_info_fail() with mock.patch.object(processutils, 'execute') as exc_mock: diff --git a/glance/tests/unit/common/test_format_inspector.py b/glance/tests/unit/common/test_format_inspector.py index 4b29c19b3..db6a9830b 100644 --- a/glance/tests/unit/common/test_format_inspector.py +++ b/glance/tests/unit/common/test_format_inspector.py @@ -16,6 +16,7 @@ import io import os import re +import struct import subprocess import tempfile from unittest import mock @@ -63,6 +64,28 @@ class TestFormatInspectors(test_utils.BaseTestCase): shell=True) return fn + def _create_allocated_vmdk(self, size_mb): + # We need a "big" VMDK file to exercise some parts of the code of the + # format_inspector. A way to create one is to first create an empty + # file, and then to convert it with the -S 0 option. + fn = tempfile.mktemp(prefix='glance-unittest-formatinspector-', + suffix='.vmdk') + self._created_files.append(fn) + zeroes = tempfile.mktemp(prefix='glance-unittest-formatinspector-', + suffix='.zero') + self._created_files.append(zeroes) + + # Create an empty file + subprocess.check_output( + 'dd if=/dev/zero of=%s bs=1M count=%i' % (zeroes, size_mb), + shell=True) + + # Convert it to VMDK + subprocess.check_output( + 'qemu-img convert -f raw -O vmdk -S 0 %s %s' % (zeroes, fn), + shell=True) + return fn + def _test_format_at_block_size(self, format_name, img, block_size): fmt = format_inspector.get_inspector(format_name)() self.assertIsNotNone(fmt, @@ -119,6 +142,64 @@ class TestFormatInspectors(test_utils.BaseTestCase): def test_vmdk(self): self._test_format('vmdk') + def test_vmdk_bad_descriptor_offset(self): + format_name = 'vmdk' + image_size = 10 * units.Mi + descriptorOffsetAddr = 0x1c + BAD_ADDRESS = 0x400 + img = self._create_img(format_name, image_size) + + # Corrupt the header + fd = open(img, 'r+b') + fd.seek(descriptorOffsetAddr) + fd.write(struct.pack('<Q', BAD_ADDRESS // 512)) + fd.close() + + # Read the format in various sizes, some of which will read whole + # sections in a single read, others will be completely unaligned, etc. + for block_size in (64 * units.Ki, 512, 17, 1 * units.Mi): + fmt = self._test_format_at_block_size(format_name, img, block_size) + self.assertTrue(fmt.format_match, + 'Failed to match %s at size %i block %i' % ( + format_name, image_size, block_size)) + self.assertEqual(0, fmt.virtual_size, + ('Calculated a virtual size for a corrupt %s at ' + 'size %i block %i') % (format_name, image_size, + block_size)) + + def test_vmdk_bad_descriptor_mem_limit(self): + format_name = 'vmdk' + image_size = 5 * units.Mi + virtual_size = 5 * units.Mi + descriptorOffsetAddr = 0x1c + descriptorSizeAddr = descriptorOffsetAddr + 8 + twoMBInSectors = (2 << 20) // 512 + # We need a big VMDK because otherwise we will not have enough data to + # fill-up the CaptureRegion. + img = self._create_allocated_vmdk(image_size // units.Mi) + + # Corrupt the end of descriptor address so it "ends" at 2MB + fd = open(img, 'r+b') + fd.seek(descriptorSizeAddr) + fd.write(struct.pack('<Q', twoMBInSectors)) + fd.close() + + # Read the format in various sizes, some of which will read whole + # sections in a single read, others will be completely unaligned, etc. + for block_size in (64 * units.Ki, 512, 17, 1 * units.Mi): + fmt = self._test_format_at_block_size(format_name, img, block_size) + self.assertTrue(fmt.format_match, + 'Failed to match %s at size %i block %i' % ( + format_name, image_size, block_size)) + self.assertEqual(virtual_size, fmt.virtual_size, + ('Failed to calculate size for %s at size %i ' + 'block %i') % (format_name, image_size, + block_size)) + memory = sum(fmt.context_info.values()) + self.assertLess(memory, 1.5 * units.Mi, + 'Format used more than 1.5MiB of memory: %s' % ( + fmt.context_info)) + def test_vdi(self): self._test_format('vdi') @@ -153,6 +234,24 @@ class TestFormatInspectors(test_utils.BaseTestCase): def test_vdi_invalid(self): self._test_format_with_invalid_data('vdi') + def test_vmdk_invalid_type(self): + fmt = format_inspector.get_inspector('vmdk')() + wrapper = format_inspector.InfoWrapper(open(__file__, 'rb'), fmt) + while True: + chunk = wrapper.read(32) + if not chunk: + break + + wrapper.close() + + fake_rgn = mock.MagicMock() + fake_rgn.complete = True + fake_rgn.data = b'foocreateType="someunknownformat"bar' + + with mock.patch.object(fmt, 'has_region', return_value=True): + with mock.patch.object(fmt, 'region', return_value=fake_rgn): + self.assertEqual(0, fmt.virtual_size) + class TestFormatInspectorInfra(test_utils.BaseTestCase): def _test_capture_region_bs(self, bs): @@ -257,3 +356,42 @@ class TestFormatInspectorInfra(test_utils.BaseTestCase): self.assertEqual(format_inspector.QcowInspector, format_inspector.get_inspector('qcow2')) self.assertIsNone(format_inspector.get_inspector('foo')) + + +class TestFormatInspectorsTargeted(test_utils.BaseTestCase): + def _make_vhd_meta(self, guid_raw, item_length): + # Meta region header, padded to 32 bytes + data = struct.pack('<8sHH', b'metadata', 0, 1) + data += b'0' * 20 + + # Metadata table entry, 16-byte GUID, 12-byte information, + # padded to 32-bytes + data += guid_raw + data += struct.pack('<III', 256, item_length, 0) + data += b'0' * 6 + + return data + + def test_vhd_table_over_limit(self): + ins = format_inspector.VHDXInspector() + meta = format_inspector.CaptureRegion(0, 0) + desired = b'012345678ABCDEF0' + # This is a poorly-crafted image that specifies a larger table size + # than is allowed + meta.data = self._make_vhd_meta(desired, 33 * 2048) + ins.new_region('metadata', meta) + new_region = ins._find_meta_entry(ins._guid(desired)) + # Make sure we clamp to our limit of 32 * 2048 + self.assertEqual( + format_inspector.VHDXInspector.VHDX_METADATA_TABLE_MAX_SIZE, + new_region.length) + + def test_vhd_table_under_limit(self): + ins = format_inspector.VHDXInspector() + meta = format_inspector.CaptureRegion(0, 0) + desired = b'012345678ABCDEF0' + meta.data = self._make_vhd_meta(desired, 16 * 2048) + ins.new_region('metadata', meta) + new_region = ins._find_meta_entry(ins._guid(desired)) + # Table size was under the limit, make sure we get it back + self.assertEqual(16 * 2048, new_region.length) diff --git a/glance/tests/unit/common/test_utils.py b/glance/tests/unit/common/test_utils.py index 15fb09c1a..a07b2709d 100644 --- a/glance/tests/unit/common/test_utils.py +++ b/glance/tests/unit/common/test_utils.py @@ -221,17 +221,6 @@ class TestUtils(test_utils.BaseTestCase): reader = utils.CooperativeReader([]) self.assertEqual(b'', reader.read()) - def test_cooperative_reader_of_iterator_stop_iteration_err(self): - """Ensure cooperative reader supports iterator backends too""" - reader = utils.CooperativeReader([l * 3 for l in '']) - chunks = [] - while True: - chunks.append(reader.read(3)) - if chunks[-1] == b'': - break - meat = b''.join(chunks) - self.assertEqual(b'', meat) - def _create_generator(self, chunk_size, max_iterations): chars = b'abc' iteration = 0 diff --git a/glance/tests/unit/v2/test_image_data_resource.py b/glance/tests/unit/v2/test_image_data_resource.py index 6a2f81ec3..3889c2f63 100644 --- a/glance/tests/unit/v2/test_image_data_resource.py +++ b/glance/tests/unit/v2/test_image_data_resource.py @@ -253,7 +253,7 @@ class TestImagesController(base.StoreClearingUnitTest): mock.call(mock.ANY, 'upload_image', mock.ANY), mock.call(mock.ANY, 'get_image', mock.ANY) ] - mock_enforce.has_calls(expected_call) + mock_enforce.assert_has_calls(expected_call) def test_upload_invalid(self): request = unit_test_utils.get_fake_request() diff --git a/releasenotes/notes/antelope-milestone-2-d89e39412f9c0334.yaml b/releasenotes/notes/antelope-milestone-2-d89e39412f9c0334.yaml new file mode 100644 index 000000000..89fe1d9d9 --- /dev/null +++ b/releasenotes/notes/antelope-milestone-2-d89e39412f9c0334.yaml @@ -0,0 +1,18 @@ +--- +fixes: + - | + Bug 1990854_: oslo_limit section not clear + - | + Bug 1779781_: virt/vmware not support VirtualSriovEthernetCard + - | + Bug 1647491_: Missing documentation for glance-manage db_purge command + - | + Bug 1983279_: Cannot upload vmdk images due to unsupported vmdk format + - | + Bug 1989268_: Wrong assertion method + + .. _1990854: https://code.launchpad.net/bugs/1990854 + .. _1779781: https://code.launchpad.net/bugs/1779781 + .. _1647491: https://code.launchpad.net/bugs/1647491 + .. _1983279: https://code.launchpad.net/bugs/1983279 + .. _1989268: https://code.launchpad.net/bugs/1989268 diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 9606f0a8d..e172c7e13 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 unreleased + zed yoga xena wallaby diff --git a/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po b/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po index 4a3512f80..15d658901 100644 --- a/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po +++ b/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po @@ -7,11 +7,11 @@ msgid "" msgstr "" "Project-Id-Version: Glance Release Notes\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2022-08-24 12:12+0000\n" +"POT-Creation-Date: 2022-10-12 14:12+0000\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"PO-Revision-Date: 2022-08-21 11:48+0000\n" +"PO-Revision-Date: 2022-10-16 10:44+0000\n" "Last-Translator: Andi Chandler <andi@gowling.com>\n" "Language-Team: English (United Kingdom)\n" "Language: en_GB\n" @@ -174,11 +174,11 @@ msgstr "24.0.0" msgid "24.1.0" msgstr "24.1.0" -msgid "24.1.0-3" -msgstr "24.1.0-3" +msgid "24.1.0-4" +msgstr "24.1.0-4" -msgid "25.0.0.0b2" -msgstr "25.0.0.0b2" +msgid "25.0.0" +msgstr "25.0.0" msgid "" "A change was added to the import API which provides time-based locking of an " @@ -621,6 +621,15 @@ msgstr "" "longer work with the Ocata glance manage db commands." msgid "" +"Also, the project reader role is ready to use. Users with reader role can " +"only perform the read-only operations within their project. This role can be " +"used for the audit purposes." +msgstr "" +"Also, the project reader role is ready to use. Users with reader role can " +"only perform the read-only operations within their project. This role can be " +"used for audit purposes." + +msgid "" "Although support has been added for Glance to be run as a WSGI application " "hosted by a web server, the atypical nature of the Images APIs provided by " "Glance, which enable transfer of copious amounts of image data, makes it " @@ -1182,6 +1191,13 @@ msgid "" msgstr "" "Bug 1895663_: Image import \"web-download\" doesn't check on download size" +msgid "" +"Bug 1905672_: Non existing property protection file raises 500 Internal " +"server error" +msgstr "" +"Bug 1905672_: Non-existing property protection file raises 500 Internal " +"server error" + msgid "Bug 1913625_: Glance will leak staging data" msgstr "Bug 1913625_: Glance will leak staging data" @@ -1301,6 +1317,9 @@ msgstr "" "Bug 1973631_: List call for metadef namespaces returns 404 not found while " "fetching resource_types" +msgid "Bug 1982426_: Python3.11: \"glance-manage\" crashes" +msgstr "Bug 1982426_: Python3.11: \"glance-manage\" crashes" + msgid "Bug Fixes" msgstr "Bug Fixes" @@ -1627,6 +1646,19 @@ msgstr "" "the Glare database." msgid "" +"For the details on what changed from the existing policy, please refer to " +"the `RBAC new guidelines`_. We have implemented only phase-1 of the `RBAC " +"new guidelines`_. Currently, scope checks and new defaults are disabled by " +"default. You can enable them by switching the below config option in " +"``glance.conf`` file::" +msgstr "" +"For the details on what changed from the existing policy, please refer to " +"the `RBAC new guidelines`_. We have implemented only phase-1 of the `RBAC " +"new guidelines`_. Currently, scope checks and new defaults are disabled by " +"default. You can enable them by switching the below config option in " +"``glance.conf`` file::" + +msgid "" "Formerly, it was possible to add members to an image whose visibility was " "``private``, thereby creating a \"shared\" image. In this release, an image " "must have a visibility of ``shared`` in order to accept member operations. " @@ -1769,6 +1801,21 @@ msgid "Glance services can now run on Windows." msgstr "Glance services can now run on Windows." msgid "" +"Glance to glance image import plugin. With this release users can import an " +"image from an other glance server from an other opensatck region. The two " +"glance services must use the same keystone service. The feature use the same " +"keystone authentication token on both glance services and copy by default " +"container_format, disk_format and customizable properties from source image " +"``['hw_', 'trait:', 'os_distro', 'os_secure_boot', 'os_type']``" +msgstr "" +"Glance to glance image import plugin. With this release, users can import an " +"image from another Glance server from another OpenStack region. The two " +"glance services must use the same keystone service. The feature use the same " +"Keystone authentication token on both Glance services and copy by default " +"container_format, disk_format and customizable properties from source image " +"``['hw_', 'trait:', 'os_distro', 'os_secure_boot', 'os_type']``" + +msgid "" "Glance uses the ``cursive`` library's functionality to verify digital " "signatures. To familiarize yourself with this new dependency and see the " "list of transitive dependencies visit http://git.openstack.org/cgit/" @@ -3005,6 +3052,15 @@ msgstr "" "findings to the Glance documentation." msgid "" +"The Glance policies have been modified to drop the system scope. Every API " +"policy is scoped to project. This means that system scoped users will get " +"403 permission denied error." +msgstr "" +"The Glance policies have been modified to drop the system scope. Every API " +"policy is scoped to the project. This means that system-scoped users will " +"get a 403 permission denied error." + +msgid "" "The Glance project team is committed to the stability of Glance. As part of " "OpenStack, we are committed to `The Four Opens`_. If the ability to run " "Glance under uWSGI is important to you, feel free to participate in the " @@ -3129,6 +3185,19 @@ msgstr "" "priorities." msgid "" +"The Zed release includes some important milestones in Glance development " +"priorities. * Extended the functionality of stores-detail API * Added glance-" +"download internal plugin to download the image from remote glance * Added " +"support for immediate caching of an image * Removed dead code of auth and " +"policy layers" +msgstr "" +"The Zed release includes some important milestones in Glance development " +"priorities. * Extended the functionality of stores-detail API * Added glance-" +"download internal plugin to download the image from remote glance * Added " +"support for immediate caching of an image * Removed dead code of auth and " +"policy layers" + +msgid "" "The `Multi-Store Backend Support <https://specs.openstack.org/openstack/" "glance-specs/specs/rocky/implemented/glance/multi-store.html>`_ feature is " "introduced on an experimental basis in the EXPERIMENTAL Image Service API " @@ -4240,6 +4309,17 @@ msgstr "" "ref <https://developer.openstack.org/api-ref/image/v2/index.html#image-" "service-info-discovery>`_." +msgid "" +"This release brings expansion in the functionality of stores-detail API. " +"The stores detail API will list the way each store is configured, whereas " +"previously this worked only for rbd store. The API remains admin-only by " +"default as it exposes backend information." +msgstr "" +"This release brings expansion in the functionality of stores-detail API. The " +"store's detail API will list the way each store is configured, whereas " +"previously this worked only for the RBD store. The API remains admin-only by " +"default as it exposes backend information." + msgid "This release has impact on API behavior." msgstr "This release has impact on API behaviour." @@ -4594,6 +4674,13 @@ msgstr "" "API." msgid "" +"We recommend to enable the both scope as well new defaults together " +"otherwise you may experience some late failures with unclear error messages." +msgstr "" +"We recommend enabling both scopes as well new defaults together otherwise " +"you may experience some late failures with unclear error messages." + +msgid "" "We renew that recommendation for the Queens release. In particular, Glance " "tasks (which are required for the interoperable image import functionality) " "do not execute when Glance is run under uWSGI (which is the OpenStack " @@ -4779,6 +4866,9 @@ msgstr "" msgid "You may upgrade these definitions using:" msgstr "You may upgrade these definitions using:" +msgid "Zed Series Release Notes" +msgstr "Zed Series Release Notes" + msgid "" "[`Community Goal <https://governance.openstack.org/tc/goals/stein/upgrade-" "checkers.html>`_] Support has been added for developers to write pre-upgrade " diff --git a/releasenotes/source/locale/fr/LC_MESSAGES/releasenotes.po b/releasenotes/source/locale/fr/LC_MESSAGES/releasenotes.po new file mode 100644 index 000000000..f74779e8a --- /dev/null +++ b/releasenotes/source/locale/fr/LC_MESSAGES/releasenotes.po @@ -0,0 +1,63 @@ +# Gérald LONLAS <g.lonlas@gmail.com>, 2016. #zanata +msgid "" +msgstr "" +"Project-Id-Version: Glance Release Notes\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2022-09-14 19:46+0000\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"PO-Revision-Date: 2016-10-22 04:47+0000\n" +"Last-Translator: Gérald LONLAS <g.lonlas@gmail.com>\n" +"Language-Team: French\n" +"Language: fr\n" +"X-Generator: Zanata 4.3.3\n" +"Plural-Forms: nplurals=2; plural=(n > 1)\n" + +msgid "11.0.1" +msgstr "11.0.1" + +msgid "12.0.0" +msgstr "12.0.0" + +msgid "13.0.0" +msgstr "13.0.0" + +msgid "Bug Fixes" +msgstr "Corrections de bugs" + +msgid "Critical Issues" +msgstr "Erreurs critiques" + +msgid "Current Series Release Notes" +msgstr "Note de la release actuelle" + +msgid "Deprecation Notes" +msgstr "Notes dépréciées " + +msgid "Glance Release Notes" +msgstr "Note de release de Glance" + +msgid "Liberty Series Release Notes" +msgstr "Note de release pour Liberty" + +msgid "Mitaka Series Release Notes" +msgstr "Note de release pour Mitaka" + +msgid "New Features" +msgstr "Nouvelles fonctionnalités" + +msgid "Newton Series Release Notes" +msgstr "Note de release pour Newton" + +msgid "Security Issues" +msgstr "Problèmes de sécurités" + +msgid "Start using reno to manage release notes." +msgstr "Commence à utiliser reno pour la gestion des notes de release" + +msgid "Translations have been synced from Zanata." +msgstr "Les traductions ont été synchronisées depuis Zanata" + +msgid "Upgrade Notes" +msgstr "Notes de mises à jours" diff --git a/releasenotes/source/zed.rst b/releasenotes/source/zed.rst new file mode 100644 index 000000000..9608c05e4 --- /dev/null +++ b/releasenotes/source/zed.rst @@ -0,0 +1,6 @@ +======================== +Zed Series Release Notes +======================== + +.. release-notes:: + :branch: stable/zed @@ -2,7 +2,6 @@ minversion = 3.18.0 # python runtimes: https://governance.openstack.org/tc/reference/runtimes/ussuri.html envlist = functional-py39,py39,pep8 -skipsdist = True skip_missing_interpreters = true # this allows tox to infer the base python from the environment name # and override any basepython configured in this file @@ -33,7 +32,9 @@ commands = allowlist_externals = bash find rm -passenv = *_proxy *_PROXY +passenv = + *_proxy + *_PROXY [testenv:functional] # this will use whatever the system python3 is @@ -66,17 +67,6 @@ basepython = python3 commands = oslopolicy-sample-generator --config-file=etc/glance-policy-generator.conf -[testenv:gateonly] -# NOTE(rosmaita): these tests catch configuration problems for some code -# constants that must be maintained manually; we have them separated out -# so they don't affect local development -# TODO(someone other than me): figure out how to make these changes either -# automatic or unnecessary -setenv = - TEST_PATH = ./glance/tests/gate -commands = - stestr run {posargs} - [testenv:pep8] commands = flake8 {posargs} |