diff options
64 files changed, 947 insertions, 431 deletions
@@ -1,3 +1,13 @@ +- nodeset: + name: trove-ubuntu-bionic + nodes: + - name: controller + label: nested-virt-ubuntu-bionic + groups: + - name: tempest + nodes: + - controller + - project: templates: - check-requirements @@ -17,6 +27,10 @@ voting: false - trove-tempest-ipv6-only: voting: false + - trove-tempest + - trove-functional-mysql: + voting: false + gate: queue: trove jobs: @@ -98,6 +112,8 @@ - job: name: trove-functional-mysql parent: trove-devstack-base + nodeset: trove-ubuntu-bionic + timeout: 10800 vars: devstack_localrc: TROVE_RESIZE_TIME_OUT: 1800 @@ -287,7 +303,8 @@ - job: name: trove-tempest parent: devstack-tempest - timeout: 7800 + nodeset: trove-ubuntu-bionic + timeout: 10800 required-projects: &base_required_projects - openstack/trove - openstack/trove-tempest-plugin @@ -300,7 +317,7 @@ - ^releasenotes/.*$ vars: &base_vars tox_envlist: all - tempest_concurrency: 2 + tempest_concurrency: 1 devstack_localrc: TEMPEST_PLUGINS: /opt/stack/trove-tempest-plugin USE_PYTHON3: true @@ -51,6 +51,6 @@ References * `Manual installation docs`_ * `Build guest image`_ -.. _Installation docs: https://docs.openstack.org/trove/latest/install/install.html -.. _Manual installation docs: https://docs.openstack.org/trove/latest/install/manual_install.html +.. _Installation docs: https://docs.openstack.org/trove/latest/install/ +.. _Manual installation docs: https://docs.openstack.org/trove/latest/install/install-manual.html .. _Build guest image: https://docs.openstack.org/trove/latest/admin/building_guest_images.html diff --git a/api-ref/source/datastore-versions.inc b/api-ref/source/datastore-versions.inc index c431b1b2..1f0daf78 100644 --- a/api-ref/source/datastore-versions.inc +++ b/api-ref/source/datastore-versions.inc @@ -275,8 +275,11 @@ Create datastore version .. rest_method:: POST /v1.0/{project_id}/mgmt/datastore-versions -Admin only API. Register a datastore version, create the datastore if doesn't -exist. +Admin only API. Register a datastore version, the datastore is created +automatically if doesn't exist. + +It's allowed to create datastore versions with the same name but different +version numbers, or vice versa. Normal response codes: 202 @@ -293,6 +296,7 @@ Request - image_tags: image_tags - active: active - default: default + - version: version_number Request Example --------------- @@ -361,6 +365,8 @@ Update datastore version details Admin only API. Update a specific datastore version. +The version number is not allowed to update. + Normal response codes: 202 Request @@ -370,6 +376,7 @@ Request - project_id: project_id - datastore_version_id: datastore_version_id + - name: datastore_version_name_optional - datastore_manager: datastore_type - image: image_id - image_tags: image_tags diff --git a/api-ref/source/instances.inc b/api-ref/source/instances.inc index 1279bc21..0cc834ec 100644 --- a/api-ref/source/instances.inc +++ b/api-ref/source/instances.inc @@ -92,11 +92,15 @@ Create database instance Creates a database instance. -Asynchronously provisions a database instance. You must specify a flavor ID, a -volume size and the tenant network ID. The service provisions the instance with -a volume of the requested size, which serves as storage for the database -instance. The database service can only be access within the tenant network, -unless the ``access`` parameter is defined. +Normally, you must specify a flavor ID, a volume size and the network the +instance is attached to. The service provisions the instance with a volume of +the requested size, which serves as storage for the database instance. + +If creating replica (a secondary instance of the replication cluster), flavor +and volume size are not needed. + +The database service can only be access within the tenant network, unless the +``access`` parameter is defined. Normal response codes: 200 @@ -115,6 +119,7 @@ Request - datastore: datastore1 - datastore.type: datastore_type - datastore.version: datastore_version + - datastore.version_number: version_number - name: instanceName1 - flavorRef: flavorRef - volume: volume @@ -166,6 +171,7 @@ Response Parameters - datastore: datastore2 - datastore.type: datastore_type - datastore.version: datastore_version1 + - datastore.version_number: version_number - volume: volume - volume.size: volume_size - volume.used: volume_used diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index 101f15d2..20a0123b 100755 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -74,7 +74,8 @@ user_project: type: string version: description: | - Name or ID of the datastore version. + Name or ID of the datastore version. If there are multiple datastore + versions with the same name but different version numbers, ID is needed. in: path required: false type: string @@ -353,10 +354,18 @@ datastore_version_id1: type: string datastore_version_name: description: | - The name of the datastore version. + The name of the datastore version. Different datastore versions can have + the same name. in: body required: true type: string +datastore_version_name_optional: + description: | + The name of the datastore version. Different datastore versions can have + the same name. + in: body + required: false + type: string default: description: | When true this datastore version is created as the default in the @@ -910,6 +919,14 @@ values: in: body required: true type: string +version_number: + description: | + The version number for the database. In container based trove instance + deployment, the version number is the same as the container image tag, + e.g. for MySQL, a valid version number is 5.7.30 + in: body + required: false + type: string volume: description: | A ``volume`` object. diff --git a/api-ref/source/samples/config-group-create-request.json b/api-ref/source/samples/config-group-create-request.json index 51472330..f19c4a98 100644 --- a/api-ref/source/samples/config-group-create-request.json +++ b/api-ref/source/samples/config-group-create-request.json @@ -3,7 +3,8 @@ "name": "group1", "datastore": { "type": "mysql", - "version": "5.7.29" + "version": "mysql-5.7", + "version_number": "5.7.29" }, "values": { "connect_timeout": 200 diff --git a/api-ref/source/samples/config-group-create-response.json b/api-ref/source/samples/config-group-create-response.json index 825ea1ab..1bbb3e5b 100644 --- a/api-ref/source/samples/config-group-create-response.json +++ b/api-ref/source/samples/config-group-create-response.json @@ -4,7 +4,8 @@ "updated": "2020-06-16T10:40:50", "datastore_name": "mysql", "datastore_version_id": "cf91aa9a-2192-4ec4-b7ce-5cac3b1e7dbe", - "datastore_version_name": "5.7.29", + "datastore_version_name": "mysql-5.7", + "datastore_version_number": "5.7.29", "description": null, "id": "9dcfca0b-d181-4b36-bbf0-09bc47b103ab", "instance_count": 0, diff --git a/api-ref/source/samples/config-group-show-response.json b/api-ref/source/samples/config-group-show-response.json index a5ee26b4..88791e4b 100644 --- a/api-ref/source/samples/config-group-show-response.json +++ b/api-ref/source/samples/config-group-show-response.json @@ -1,16 +1,17 @@ { "configuration": { "datastore_name": "mysql", + "datastore_version_id": "b9f97132-467b-4f8e-b12d-947cfc223ac3", + "datastore_version_name": "mysql-5.7", + "datastore_version_number": "5.7.29", "updated": "2015-11-22T19:07:20", "values": { "connect_timeout": 17 }, "name": "group1", "created": "2015-11-20T20:51:24", - "datastore_version_name": "5.6", "instance_count": 1, "id": "1c8a4fdd-690c-4e6e-b2e1-148b8d738770", - "datastore_version_id": "b9f97132-467b-4f8e-b12d-947cfc223ac3", "description": null } } diff --git a/api-ref/source/samples/config-groups-list-response.json b/api-ref/source/samples/config-groups-list-response.json index 5bdaa990..bcedcb16 100644 --- a/api-ref/source/samples/config-groups-list-response.json +++ b/api-ref/source/samples/config-groups-list-response.json @@ -1,13 +1,14 @@ { "configurations": [ { - "datastore_name": "mysql", "updated": "2015-07-01T16:38:27", + "datastore_version_id": "2dc7faa0-efff-4c2b-8cff-bcd949c518a5", + "datastore_name": "mysql", + "datastore_version_name": "mysql-5.7", + "datastore_version_number": "5.7.29", "name": "group1", "created": "2015-07-01T16:38:27", - "datastore_version_name": "5.6", "id": "2aa51628-5c42-4086-8682-137caffd2ba6", - "datastore_version_id": "2dc7faa0-efff-4c2b-8cff-bcd949c518a5", "description": null } ] diff --git a/api-ref/source/samples/datastore-version-create-request.json b/api-ref/source/samples/datastore-version-create-request.json index 9107eb63..32f4d26d 100644 --- a/api-ref/source/samples/datastore-version-create-request.json +++ b/api-ref/source/samples/datastore-version-create-request.json @@ -3,9 +3,10 @@ "datastore_name": "mysql", "datastore_manager": "mysql", "name": "test", - "image": "58b83318-cb18-4189-8d89-a015dc3839dd", + "image_tags": ["trove"], "active": true, "default": false, - "packages": [] + "packages": [], + "version": "5.7.30" } }
\ No newline at end of file diff --git a/api-ref/source/samples/datastore-version-list-response.json b/api-ref/source/samples/datastore-version-list-response.json index 4c20d87a..3b526914 100644 --- a/api-ref/source/samples/datastore-version-list-response.json +++ b/api-ref/source/samples/datastore-version-list-response.json @@ -13,7 +13,8 @@ "rel": "bookmark" } ], - "name": "12" + "name": "12", + "version": "5.7.29" } ] }
\ No newline at end of file diff --git a/api-ref/source/samples/datastore-version-mgmt-list-response.json b/api-ref/source/samples/datastore-version-mgmt-list-response.json index 47e363fd..51074a12 100644 --- a/api-ref/source/samples/datastore-version-mgmt-list-response.json +++ b/api-ref/source/samples/datastore-version-mgmt-list-response.json @@ -11,7 +11,8 @@ "name": "10.4-dev-train", "packages": [ "" - ] + ], + "version": "10.4.12" }, { "active": true, @@ -24,7 +25,8 @@ "name": "12", "packages": [ "" - ] + ], + "version": "12.4" } ] }
\ No newline at end of file diff --git a/api-ref/source/samples/datastore-version-mgmt-patch-request.json b/api-ref/source/samples/datastore-version-mgmt-patch-request.json index 85b98a22..c1a30981 100644 --- a/api-ref/source/samples/datastore-version-mgmt-patch-request.json +++ b/api-ref/source/samples/datastore-version-mgmt-patch-request.json @@ -1,4 +1,5 @@ { + "name": "new name", "image": "42706631-3b76-4d1c-95c9-6a85e72eebda", "active": true }
\ No newline at end of file diff --git a/api-ref/source/samples/datastore-version-mgmt-show-response.json b/api-ref/source/samples/datastore-version-mgmt-show-response.json index 8f1b78bf..3535d480 100644 --- a/api-ref/source/samples/datastore-version-mgmt-show-response.json +++ b/api-ref/source/samples/datastore-version-mgmt-show-response.json @@ -10,6 +10,7 @@ "name": "10.4-dev-train", "packages": [ "" - ] + ], + "version": "10.4.12" } }
\ No newline at end of file diff --git a/api-ref/source/samples/datastore-version-show-response.json b/api-ref/source/samples/datastore-version-show-response.json index f1798820..bf6ca44e 100644 --- a/api-ref/source/samples/datastore-version-show-response.json +++ b/api-ref/source/samples/datastore-version-show-response.json @@ -12,6 +12,7 @@ "rel": "bookmark" } ], - "name": "12" + "name": "12", + "version": "5.7.29" } }
\ No newline at end of file diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 3d45bf84..dffe0298 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -180,8 +180,6 @@ function config_trove_apache_wsgi { s|%APIWORKERS%|${API_WORKERS}|g; " -i ${trove_apache_conf} enable_apache_site trove-api - tail_log trove-access /var/log/${APACHE_NAME}/trove-api-access.log - tail_log trove-api /var/log/${APACHE_NAME}/trove-api.log } # configure_trove() - Set config files, create data dirs, etc @@ -480,11 +478,13 @@ function create_guest_image { --tag trove \ --property hw_rng_model='virtio' \ --file ${image_file} \ + --debug \ -c id -f value) + echo "Glance image ${glance_image_id} uploaded" echo "Register the image in datastore" $TROVE_MANAGE datastore_update $TROVE_DATASTORE_TYPE "" - $TROVE_MANAGE datastore_version_update $TROVE_DATASTORE_TYPE $TROVE_DATASTORE_VERSION $TROVE_DATASTORE_TYPE $glance_image_id "trove" "" 1 + $TROVE_MANAGE datastore_version_update $TROVE_DATASTORE_TYPE $TROVE_DATASTORE_VERSION $TROVE_DATASTORE_TYPE "" "" 1 --image-tags trove $TROVE_MANAGE datastore_update $TROVE_DATASTORE_TYPE $TROVE_DATASTORE_VERSION echo "Add parameter validation rules if available" diff --git a/doc/source/admin/building_guest_images.rst b/doc/source/admin/building_guest_images.rst index df5f49ee..ce56520a 100644 --- a/doc/source/admin/building_guest_images.rst +++ b/doc/source/admin/building_guest_images.rst @@ -162,9 +162,8 @@ The trove guest image could be created by running the following command: rebuilt which is convenient for debugging. Trove guest agent will ssh into the controller node and download trove code during the service initialization. -* if ``dev_mode=false``, the trove code for guest agent is injected into the - image at the building time. Now ``dev_mode=false`` is still in experimental - and not considered production ready yet. +* If ``dev_mode=false``, the trove code for guest agent is injected into the + image at the building time. * Some other global variables: diff --git a/doc/source/admin/datastore.rst b/doc/source/admin/datastore.rst index 98f0922e..ddac4b8d 100644 --- a/doc/source/admin/datastore.rst +++ b/doc/source/admin/datastore.rst @@ -16,9 +16,9 @@ database, Trove could support 5.7.29, 5.7.30 or 5.8, etc. .. note:: - Starting from Victoria, the datastore version name must be the same with the - image tag of the specific database. To support MySQL 5.7.29, a new datastore - version named 5.7.29 based on `mysql docker image + Starting from Victoria, the datastore version number must be the same with + the image tag of the specific database. To support MySQL 5.7.29, a new + datastore version with version number 5.7.29 based on `mysql docker image <https://hub.docker.com/_/mysql?tab=tags&name=5.7.29>`_ needs to be created. A datastore version is always associated with a Glance image, either by image @@ -32,7 +32,8 @@ Create datastore version ~~~~~~~~~~~~~~~~~~~~~~~~ When creating a datastore version, Trove will create the datastore first if it -doesn't exist. +doesn't exist. Different datastore versions can have the same name but +different version numbers, or same version number but different names. When using image tags, make sure the image with the tags exists before creating the datastore version. @@ -52,6 +53,8 @@ To create a datastore version: #. Register image with Image service You need to register your guest image with the Image service as cloud admin. + In this example, the image is assigned tags that will be used when creating + datastore version. .. code-block:: console @@ -69,7 +72,8 @@ To create a datastore version: openstack datastore version create 5.7.29 mysql mysql "" \ --image-tags trove,mysql \ - --active --default + --active --default \ + --version-number 5.7.29 #. Load validation rules for configuration groups diff --git a/doc/source/admin/run_trove_in_production.rst b/doc/source/admin/run_trove_in_production.rst index 5bdf36d9..5cfde6b3 100644 --- a/doc/source/admin/run_trove_in_production.rst +++ b/doc/source/admin/run_trove_in_production.rst @@ -334,7 +334,8 @@ Command examples: $ # Creating datastore 'mysql' and datastore version 5.7.29. $ openstack datastore version create 5.7.29 mysql mysql "" \ --image-tags trove,mysql \ - --active --default + --active --default \ + --version-number 5.7.29 $ # Register configuration parameters for the datastore version $ trove-manage db_load_datastore_config_parameters mysql 5.7.29 ${trove_repo_dir}}/trove/templates/mysql/validation-rules.json diff --git a/doc/source/admin/troubleshooting.rst b/doc/source/admin/troubleshooting.rst index ea56a464..be9f494c 100644 --- a/doc/source/admin/troubleshooting.rst +++ b/doc/source/admin/troubleshooting.rst @@ -46,7 +46,7 @@ After log into the instance, you can check the trove-guestagent log by: .. code-block:: console - sudo journalctl -u trove-guest.service | less # or + sudo journalctl -u guest-agent.service | less # or sudo vi /var/log/trove/trove-guestagent.log Please contact Trove team in #openstack-trove IRC channel or send email to diff --git a/doc/source/admin/upgrade.rst b/doc/source/admin/upgrade.rst index 1e4946ca..757c06fd 100644 --- a/doc/source/admin/upgrade.rst +++ b/doc/source/admin/upgrade.rst @@ -157,7 +157,7 @@ Upgrade Trove services --property hw_rng_model='virtio' \ --tag trove \ -c id -f value) - $ trove-manage datastore_version_update mysql 5.7.29 mysql $imageid "" "" 1 + $ trove-manage datastore_version_update mysql 5.7.29 mysql $imageid "" 1 $ trove-manage db_load_datastore_config_parameters mysql 5.7.29 $stackdir/trove/trove/templates/mysql/validation-rules.json Upgrade Trove guest agent diff --git a/doc/source/cli/trove-manage.rst b/doc/source/cli/trove-manage.rst index ef2bb8a7..2e890739 100644 --- a/doc/source/cli/trove-manage.rst +++ b/doc/source/cli/trove-manage.rst @@ -200,7 +200,8 @@ trove-manage datastore_version_update usage: trove-manage datastore_version_update [-h] datastore version_name manager - image_id image_tags packages active + image_id packages active + --image-tags <image_tags> Add or update a datastore version. If the datastore version already exists, all values except the datastore name and version will be updated. @@ -219,13 +220,6 @@ all values except the datastore name and version will be updated. ``image_id`` ID of the image used to create an instance of the datastore version. -``image_tags`` - List of image tags separated by comma. If the image ID is not provided - explicitly, the image can be retrieved by the image tags. Multiple image tags - are separated by comma, e.g. trove,mysql. Using image tags is more flexible - than ID especially when new guest image is uploaded to Glance, Trove can pick - up the latest image automatically for creating instances. - ``packages`` Packages required by the datastore version that are installed on the guest image. @@ -239,6 +233,13 @@ all values except the datastore name and version will be updated. ``-h, --help`` show this help message and exit +``--image-tags`` + List of image tags separated by comma. If the image ID is not provided + explicitly, the image can be retrieved by the image tags. Multiple image tags + are separated by comma, e.g. trove,mysql. Using image tags is more flexible + than ID especially when new guest image is uploaded to Glance, Trove can pick + up the latest image automatically for creating instances. + trove-manage db_downgrade ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/integration/README.md b/integration/README.md index 02dcaf35..157cc3e6 100644 --- a/integration/README.md +++ b/integration/README.md @@ -133,8 +133,7 @@ PATH_DEVSTACK_OUTPUT=/opt/stack \ - If the script is running as a part of DevStack, the viriable `PATH_DEVSTACK_OUTPUT` is set automatically. - if `dev_mode=false`, the trove code for guest agent is injected into the - image at the building time. Now `dev_mode=false` is still in experimental - and not considered production ready yet. + image at the building time. - If `dev_mode=true`, no Trove code is injected into the guest image. The guest agent will download Trove code during the service initialization. diff --git a/integration/scripts/files/elements/guest-agent/element-deps b/integration/scripts/files/elements/guest-agent/element-deps index ef309837..6dcd66dc 100644 --- a/integration/scripts/files/elements/guest-agent/element-deps +++ b/integration/scripts/files/elements/guest-agent/element-deps @@ -4,4 +4,3 @@ pkg-map source-repositories svc-map pip-and-virtualenv -ubuntu-docker diff --git a/integration/scripts/files/elements/guest-agent/package-installs.yaml b/integration/scripts/files/elements/guest-agent/package-installs.yaml index 37e7daa4..1cf7179f 100644 --- a/integration/scripts/files/elements/guest-agent/package-installs.yaml +++ b/integration/scripts/files/elements/guest-agent/package-installs.yaml @@ -1,15 +1,42 @@ guest-agent: installtype: package +acl: +acpid: + arch: i386, amd64, arm64, s390x +apparmor: +apt-transport-https: build-essential: -python3-all: -python3-all-dev: -python3-pip: -python3-sqlalchemy: +cloud-guest-utils: +cloud-init: +cron: +dbus: +dkms: +dmeventd: +ethtool: +gpg-agent: +ifenslave: +ifupdown: +iptables: +isc-dhcp-client: libxml2-dev: libxslt1-dev: libffi-dev: libssl-dev: libyaml-dev: +less: +logrotate: +netbase: +open-vm-tools: + arch: i386, amd64 openssh-client: openssh-server: +pollinate: +psmisc: +python3-sqlalchemy: rsync: +rsyslog: +ubuntu-cloudimage-keyring: +ureadahead: +uuid-runtime: +vim-tiny: +vlan: diff --git a/integration/scripts/files/elements/guest-agent/post-install.d/99-clean-apt b/integration/scripts/files/elements/guest-agent/post-install.d/99-clean-apt index 227c508d..35446f00 100755 --- a/integration/scripts/files/elements/guest-agent/post-install.d/99-clean-apt +++ b/integration/scripts/files/elements/guest-agent/post-install.d/99-clean-apt @@ -6,4 +6,4 @@ set -e set -o xtrace -apt-get clean +apt-get --assume-yes purge --auto-remove diff --git a/integration/scripts/files/elements/ubuntu-docker/install.d/21-docker b/integration/scripts/files/elements/ubuntu-docker/install.d/21-docker index 44041384..5b8fe7fc 100755 --- a/integration/scripts/files/elements/ubuntu-docker/install.d/21-docker +++ b/integration/scripts/files/elements/ubuntu-docker/install.d/21-docker @@ -14,6 +14,7 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu ${DIB_RELEASE} stable" apt-get update apt-get install -y -qq docker-ce >/dev/null +apt-get clean echo "Adding ${GUEST_USERNAME} user to docker group" usermod -aG docker ${GUEST_USERNAME} diff --git a/integration/scripts/functions_qemu b/integration/scripts/functions_qemu index 12ff4f20..aa88f559 100644 --- a/integration/scripts/functions_qemu +++ b/integration/scripts/functions_qemu @@ -15,8 +15,14 @@ function build_guest_image() { local working_dir=$(dirname ${image_output}) local root_password=${TROVE_ROOT_PASSWORD} - local elementes="base vm" local trove_elements_path=${PATH_TROVE}/integration/scripts/files/elements + # For system-wide installs, DIB will automatically find the elements, so we only check local path + if [[ "${DIB_LOCAL_ELEMENTS_PATH}" ]]; then + export ELEMENTS_PATH=${trove_elements_path}:${DIB_LOCAL_ELEMENTS_PATH} + else + export ELEMENTS_PATH=${trove_elements_path} + fi + local GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-3} local GUEST_CACHEDIR=${GUEST_CACHEDIR:-"$HOME/.cache/image-create"} sudo rm -rf ${GUEST_CACHEDIR} @@ -33,29 +39,29 @@ function build_guest_image() { manage_ssh_keys fi - # For system-wide installs, DIB will automatically find the elements, so we only check local path - if [[ "${DIB_LOCAL_ELEMENTS_PATH}" ]]; then - export ELEMENTS_PATH=${trove_elements_path}:${DIB_LOCAL_ELEMENTS_PATH} - else - export ELEMENTS_PATH=${trove_elements_path} - fi + TEMP=$(mktemp -d ${working_dir}/diskimage-create.XXXXXXX) + pushd $TEMP > /dev/null - export DIB_RELEASE=${guest_release} - export DIB_CLOUD_INIT_DATASOURCES="ConfigDrive" + # Prepare elements for diskimage-builder export DIB_CLOUD_INIT_ETC_HOSTS="localhost" + local elementes="base vm" - # https://cloud-images.ubuntu.com/releases is more stable than the daily - # builds (https://cloud-images.ubuntu.com/xenial/current/), - # e.g. sometimes SHA256SUMS file is missing in the daily builds website. - # Ref: diskimage_builder/elements/ubuntu/root.d/10-cache-ubuntu-tarball - declare -A image_file_mapping=( ["xenial"]="ubuntu-16.04-server-cloudimg-amd64-root.tar.gz" ["bionic"]="ubuntu-18.04-server-cloudimg-amd64.squashfs" ) - export DIB_CLOUD_IMAGES="https://cloud-images.ubuntu.com/releases/${DIB_RELEASE}/release/" - export BASE_IMAGE_FILE=${image_file_mapping[${DIB_RELEASE}]} + # Only support ubuntu at the moment. + if [[ "${guest_os}" == "ubuntu" ]]; then + export DIB_RELEASE=${guest_release} + # https://cloud-images.ubuntu.com/releases is more stable than the daily + # builds (https://cloud-images.ubuntu.com/xenial/current/), + # e.g. sometimes SHA256SUMS file is missing in the daily builds website. + # Ref: diskimage_builder/elements/ubuntu/root.d/10-cache-ubuntu-tarball + declare -A image_file_mapping=( ["xenial"]="ubuntu-16.04-server-cloudimg-amd64-root.tar.gz" ["bionic"]="ubuntu-18.04-server-cloudimg-amd64.squashfs" ) + export DIB_CLOUD_IMAGES="https://cloud-images.ubuntu.com/releases/${DIB_RELEASE}/release/" + export BASE_IMAGE_FILE=${image_file_mapping[${DIB_RELEASE}]} + elementes="$elementes ubuntu-minimal" + fi - TEMP=$(mktemp -d ${working_dir}/diskimage-create.XXXXXXX) - pushd $TEMP > /dev/null + export DIB_CLOUD_INIT_DATASOURCES=${DIB_CLOUD_INIT_DATASOURCES:-"ConfigDrive"} + elementes="$elementes cloud-init-datasources" - elementes="$elementes ${guest_os}" elementes="$elementes pip-and-virtualenv" elementes="$elementes pip-cache" elementes="$elementes guest-agent" diff --git a/integration/scripts/trovestack b/integration/scripts/trovestack index 4e6742a1..0045fff5 100755 --- a/integration/scripts/trovestack +++ b/integration/scripts/trovestack @@ -521,11 +521,9 @@ function set_bin_path() { } function cmd_set_datastore() { - local IMAGEID=$1 - rd_manage datastore_update "$datastore" "" - # trove-manage datastore_version_update <datastore_name> <version_name> <datastore_manager> <image_id> <image_tags> <packages> <active> - rd_manage datastore_version_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "${DATASTORE_TYPE}" ${IMAGEID} "trove" "" 1 + # Use image tags for datastore version. + rd_manage datastore_version_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "${DATASTORE_TYPE}" "" "" 1 --image-tags trove rd_manage datastore_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" if [[ -f "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json ]]; then @@ -781,7 +779,7 @@ function cmd_build_and_upload_image() { exclaim "Using Glance image ID: $glance_imageid" exclaim "Updating Datastores" - cmd_set_datastore "${glance_imageid}" + cmd_set_datastore } diff --git a/lower-constraints.txt b/lower-constraints.txt index ebd0d707..b504edcc 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -106,7 +106,7 @@ pyinotify==0.9.6 pylint==1.9.2 # GPLv2 pymongo==3.0.2 PyMySQL==0.7.6 -pyOpenSSL==17.5.0 +pyOpenSSL==19.1.0 pyparsing==2.2.0 pyperclip==1.6.0 python-cinderclient==3.3.0 diff --git a/releasenotes/notes/wallaby-add-ram-quota-d8e64d0385b1429f.yaml b/releasenotes/notes/wallaby-add-ram-quota-d8e64d0385b1429f.yaml new file mode 100644 index 00000000..951d1ac3 --- /dev/null +++ b/releasenotes/notes/wallaby-add-ram-quota-d8e64d0385b1429f.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Added the ability to quota on total amount of RAM in MB used per project. + Set ``quota.max_ram_per_tenant`` to enable. Default is -1 (unlimited) + to be backwards compatible. Existing installations will need to manually + backfill quote usage for this to work as expected. diff --git a/releasenotes/notes/wallaby-fix-deleting-volume.yaml b/releasenotes/notes/wallaby-fix-deleting-volume.yaml new file mode 100644 index 00000000..1e1d1844 --- /dev/null +++ b/releasenotes/notes/wallaby-fix-deleting-volume.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - Fixed an issue that orphan volumes left after removing instances, + especially for the case that instance creation failed because of timeout + when waiting for the volume available. diff --git a/releasenotes/notes/wallaby-fix-race-condition-create-delete.yaml b/releasenotes/notes/wallaby-fix-race-condition-create-delete.yaml new file mode 100644 index 00000000..9bd41f0e --- /dev/null +++ b/releasenotes/notes/wallaby-fix-race-condition-create-delete.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - Fixed a race condition that instance becomes ERROR when Trove is handling + creating and deleting at the same time. diff --git a/trove/cmd/manage.py b/trove/cmd/manage.py index 52a342ec..c6e3c675 100644 --- a/trove/cmd/manage.py +++ b/trove/cmd/manage.py @@ -62,15 +62,18 @@ class Commands(object): print(e) def datastore_version_update(self, datastore, version_name, manager, - image_id, image_tags, packages, active): + image_id, packages, active, image_tags=None, + version=None): try: datastore_models.update_datastore_version(datastore, version_name, manager, image_id, image_tags, - packages, active) - print("Datastore version '%s' updated." % version_name) + packages, active, + version=version) + print("Datastore version '%s(%s)' updated." % + (version_name, version or version_name)) except exception.DatastoreNotFound as e: print(e) @@ -82,77 +85,113 @@ class Commands(object): def db_load_datastore_config_parameters(self, datastore, datastore_version_name, - config_file_location): + config_file_location, + version=None): print("Loading config parameters for datastore (%s) version (%s)" % (datastore, datastore_version_name)) config_models.load_datastore_configuration_parameters( - datastore, datastore_version_name, config_file_location) + datastore, datastore_version_name, config_file_location, + version_number=version) def db_remove_datastore_config_parameters(self, datastore, - datastore_version_name): + datastore_version_name, + version=None): print("Removing config parameters for datastore (%s) version (%s)" % (datastore, datastore_version_name)) config_models.remove_datastore_configuration_parameters( - datastore, datastore_version_name) + datastore, datastore_version_name, version_number=version) def datastore_version_flavor_add(self, datastore_name, - datastore_version_name, flavor_ids): + datastore_version_name, flavor_ids, + version=None): """Adds flavors for a given datastore version id.""" + dsmetadata = datastore_models.DatastoreVersionMetadata try: - dsmetadata = datastore_models.DatastoreVersionMetadata + datastore_version_id = dsmetadata.datastore_version_find( + datastore_name, + datastore_version_name, + version_number=version) + dsmetadata.add_datastore_version_flavor_association( - datastore_name, datastore_version_name, flavor_ids.split(",")) + datastore_version_id, flavor_ids.split(",")) print("Added flavors '%s' to the '%s' '%s'." % (flavor_ids, datastore_name, datastore_version_name)) - except exception.DatastoreVersionNotFound as e: + except Exception as e: print(e) def datastore_version_flavor_delete(self, datastore_name, - datastore_version_name, flavor_id): + datastore_version_name, flavor_id, + version=None): """Deletes a flavor's association with a given datastore.""" try: dsmetadata = datastore_models.DatastoreVersionMetadata + + datastore_version_id = dsmetadata.datastore_version_find( + datastore_name, + datastore_version_name, + version_number=version) + dsmetadata.delete_datastore_version_flavor_association( - datastore_name, datastore_version_name, flavor_id) + datastore_version_id, flavor_id) print("Deleted flavor '%s' from '%s' '%s'." % (flavor_id, datastore_name, datastore_version_name)) - except exception.DatastoreVersionNotFound as e: + except Exception as e: print(e) def datastore_version_volume_type_add(self, datastore_name, datastore_version_name, - volume_type_ids): + volume_type_ids, version=None): """Adds volume type assiciation for a given datastore version id.""" try: dsmetadata = datastore_models.DatastoreVersionMetadata + + datastore_version_id = dsmetadata.datastore_version_find( + datastore_name, + datastore_version_name, + version_number=version) + dsmetadata.add_datastore_version_volume_type_association( - datastore_name, datastore_version_name, + datastore_version_id, volume_type_ids.split(",")) print("Added volume type '%s' to the '%s' '%s'." % (volume_type_ids, datastore_name, datastore_version_name)) - except exception.DatastoreVersionNotFound as e: + except Exception as e: print(e) def datastore_version_volume_type_delete(self, datastore_name, datastore_version_name, - volume_type_id): + volume_type_id, version=None): """Deletes a volume type association with a given datastore.""" try: dsmetadata = datastore_models.DatastoreVersionMetadata + + datastore_version_id = dsmetadata.datastore_version_find( + datastore_name, + datastore_version_name, + version_number=version) + dsmetadata.delete_datastore_version_volume_type_association( - datastore_name, datastore_version_name, volume_type_id) + datastore_version_id, volume_type_id) print("Deleted volume type '%s' from '%s' '%s'." % (volume_type_id, datastore_name, datastore_version_name)) - except exception.DatastoreVersionNotFound as e: + except Exception as e: print(e) def datastore_version_volume_type_list(self, datastore_name, - datastore_version_name): + datastore_version_name, + version=None): """Lists volume type association with a given datastore.""" try: dsmetadata = datastore_models.DatastoreVersionMetadata - vtlist = dsmetadata.list_datastore_volume_type_associations( - datastore_name, datastore_version_name) + + datastore_version_id = dsmetadata.datastore_version_find( + datastore_name, + datastore_version_name, + version_number=version) + + vtlist = dsmetadata. \ + list_datastore_version_volume_type_associations( + datastore_version_id) if vtlist.count() > 0: for volume_type in vtlist: print("Datastore: %s, Version: %s, Volume Type: %s" % @@ -162,7 +201,7 @@ class Commands(object): print("No Volume Type Associations found for Datastore: %s, " "Version: %s." % (datastore_name, datastore_version_name)) - except exception.DatastoreVersionNotFound as e: + except Exception as e: print(e) def params_of(self, command_name): @@ -213,16 +252,20 @@ def main(): help='ID of the image used to create an instance of ' 'the datastore version.') parser.add_argument( - 'image_tags', - help='List of image tags separated by comma used for getting ' - 'guest image.') - parser.add_argument( 'packages', help='Packages required by the datastore version that ' 'are installed on the guest image.') parser.add_argument( 'active', type=int, help='Whether the datastore version is active or not. ' 'Accepted values are 0 and 1.') + parser.add_argument( + '--image-tags', + help='List of image tags separated by comma used for getting ' + 'guest image.') + parser.add_argument( + '--version', + help='The version number of the datastore version, e.g. 5.7.30. ' + 'If not specified, use <version_name> as default value.') parser = subparser.add_parser( 'db_recreate', description='Drop the database and recreate it.') @@ -242,6 +285,11 @@ def main(): 'config_file_location', help='Fully qualified file path to the configuration group ' 'parameter validation rules.') + parser.add_argument( + '--version', + help='The version number of the datastore version, e.g. 5.7.30. ' + 'If not specified, use <datastore_version_name> as default ' + 'value.') parser = subparser.add_parser( 'db_remove_datastore_config_parameters', @@ -253,51 +301,86 @@ def main(): parser.add_argument( 'datastore_version_name', help='Name of the datastore version.') + parser.add_argument( + '--version', + help='The version number of the datastore version, e.g. 5.7.30. ' + 'If not specified, use <datastore_version_name> as default ' + 'value.') parser = subparser.add_parser( - 'datastore_version_flavor_add', help='Adds flavor association to ' - 'a given datastore and datastore version.') + 'datastore_version_flavor_add', + help='Adds flavor association to a given datastore and datastore ' + 'version.') parser.add_argument('datastore_name', help='Name of the datastore.') parser.add_argument('datastore_version_name', help='Name of the ' 'datastore version.') parser.add_argument('flavor_ids', help='Comma separated list of ' 'flavor ids.') + parser.add_argument( + '--version', + help='The version number of the datastore version, e.g. 5.7.30. ' + 'If not specified, use <datastore_version_name> as default ' + 'value.') parser = subparser.add_parser( - 'datastore_version_flavor_delete', help='Deletes a flavor ' - 'associated with a given datastore and datastore version.') + 'datastore_version_flavor_delete', + help='Deletes a flavor associated with a given datastore and ' + 'datastore version.') parser.add_argument('datastore_name', help='Name of the datastore.') parser.add_argument('datastore_version_name', help='Name of the ' 'datastore version.') parser.add_argument('flavor_id', help='The flavor to be deleted for ' 'a given datastore and datastore version.') + parser.add_argument( + '--version', + help='The version number of the datastore version, e.g. 5.7.30. ' + 'If not specified, use <datastore_version_name> as default ' + 'value.') + parser = subparser.add_parser( - 'datastore_version_volume_type_add', help='Adds volume_type ' - 'association to a given datastore and datastore version.') + 'datastore_version_volume_type_add', + help='Adds volume_type association to a given datastore and ' + 'datastore version.') parser.add_argument('datastore_name', help='Name of the datastore.') parser.add_argument('datastore_version_name', help='Name of the ' 'datastore version.') parser.add_argument('volume_type_ids', help='Comma separated list of ' 'volume_type ids.') + parser.add_argument( + '--version', + help='The version number of the datastore version, e.g. 5.7.30. ' + 'If not specified, use <datastore_version_name> as default ' + 'value.') parser = subparser.add_parser( 'datastore_version_volume_type_delete', help='Deletes a volume_type ' - 'associated with a given datastore and datastore version.') + 'associated with a given datastore and datastore version.') parser.add_argument('datastore_name', help='Name of the datastore.') parser.add_argument('datastore_version_name', help='Name of the ' 'datastore version.') parser.add_argument('volume_type_id', help='The volume_type to be ' 'deleted for a given datastore and datastore ' 'version.') + parser.add_argument( + '--version', + help='The version number of the datastore version, e.g. 5.7.30. ' + 'If not specified, use <datastore_version_name> as default ' + 'value.') parser = subparser.add_parser( 'datastore_version_volume_type_list', help='Lists the volume_types ' - 'associated with a given datastore and datastore version.') + 'associated with a given datastore and datastore version.') parser.add_argument('datastore_name', help='Name of the datastore.') parser.add_argument('datastore_version_name', help='Name of the ' 'datastore version.') + parser.add_argument( + '--version', + help='The version number of the datastore version, e.g. 5.7.30. ' + 'If not specified, use <datastore_version_name> as default ' + 'value.') + cfg.custom_parser('action', actions) cfg.parse_args(sys.argv) diff --git a/trove/common/apischema.py b/trove/common/apischema.py index 9af0e19c..4c799992 100644 --- a/trove/common/apischema.py +++ b/trove/common/apischema.py @@ -410,7 +410,8 @@ instance = { "additionalProperties": True, "properties": { "type": non_empty_string, - "version": non_empty_string + "version": non_empty_string, + "version_number": non_empty_string } }, "nics": nics, @@ -820,7 +821,8 @@ configuration = { "additionalProperties": True, "properties": { "type": non_empty_string, - "version": non_empty_string + "version": non_empty_string, + "version_number": non_empty_string } } } @@ -979,7 +981,8 @@ mgmt_datastore_version = { "image": uuid, "image_tags": image_tags, "active": {"enum": [True, False]}, - "default": {"enum": [True, False]} + "default": {"enum": [True, False]}, + "version": non_empty_string } } } @@ -996,6 +999,7 @@ mgmt_datastore_version = { "image_tags": image_tags, "active": {"enum": [True, False]}, "default": {"enum": [True, False]}, + "name": non_empty_string } } } diff --git a/trove/common/cfg.py b/trove/common/cfg.py index 6109a359..9d30af17 100644 --- a/trove/common/cfg.py +++ b/trove/common/cfg.py @@ -221,6 +221,9 @@ common_opts = [ default=10, help='Default maximum number of instances per tenant.', deprecated_name='max_instances_per_user'), + cfg.IntOpt('max_ram_per_tenant', + default=-1, + help='Default maximum total amount of RAM in MB per tenant.'), cfg.IntOpt('max_accepted_volume_size', default=10, help='Default maximum volume size (in GB) for an instance.'), cfg.IntOpt('max_volumes_per_tenant', default=40, diff --git a/trove/common/exception.py b/trove/common/exception.py index 40d22463..d4b44b0c 100644 --- a/trove/common/exception.py +++ b/trove/common/exception.py @@ -134,38 +134,38 @@ class DatastoresNotFound(NotFound): class DatastoreFlavorAssociationNotFound(NotFound): message = _("Flavor %(id)s is not supported for datastore " - "%(datastore)s version %(datastore_version)s") + "version %(datastore_version_id)s") class DatastoreFlavorAssociationAlreadyExists(TroveError): message = _("Flavor %(id)s is already associated with " - "datastore %(datastore)s version %(datastore_version)s") + "datastore version %(datastore_version_id)s") class DatastoreVolumeTypeAssociationNotFound(NotFound): message = _("The volume type %(id)s is not valid for datastore " - "%(datastore)s and version %(version_id)s.") + "version %(datastore_version_id)s.") class DatastoreVolumeTypeAssociationAlreadyExists(TroveError): - message = _("Datastore '%(datastore)s' version %(datastore_version)s " + message = _("Datastore version %(datastore_version_id)s " "and volume-type %(id)s mapping already exists.") class DataStoreVersionVolumeTypeRequired(TroveError): message = _("Only specific volume types are allowed for a " - "datastore %(datastore)s version %(datastore_version)s. " + "datastore version %(datastore_version_id)s. " "You must specify a valid volume type.") class DatastoreVersionNoVolumeTypes(TroveError): message = _("No valid volume types could be found for datastore " - "%(datastore)s and version %(datastore_version)s.") + "version %(datastore_version_id)s.") class DatastoreNoVersion(TroveError): @@ -180,7 +180,8 @@ class DatastoreVersionInactive(TroveError): class DatastoreVersionAlreadyExists(BadRequest): - message = _("A datastore version with the name '%(name)s' already exists.") + message = _("The datastore version '%(name)s(%(version)s)' already " + "exists.") class DatastoreVersionsExist(BadRequest): @@ -193,6 +194,13 @@ class DatastoreVersionsInUse(BadRequest): message = _("Datastore version is in use by %(resource)s.") +class DatastoreVersionsNoUniqueMatch(TroveError): + + message = _("Multiple datastore versions found for '%(name)s', " + "use an UUID or specify both the name and version number to " + "be more specific.") + + class DatastoreDefaultDatastoreNotFound(TroveError): message = _("Please specify datastore. Default datastore " @@ -221,12 +229,6 @@ class DatastoreOperationNotSupported(TroveError): "the '%(datastore)s' datastore.") -class NoUniqueMatch(TroveError): - - message = _("Multiple matches found for '%(name)s', " - "use an UUID to be more specific.") - - class OverLimit(TroveError): # internal_message is used for log, stop translating. diff --git a/trove/configuration/models.py b/trove/configuration/models.py index 294ef42c..8ec97477 100644 --- a/trove/configuration/models.py +++ b/trove/configuration/models.py @@ -354,17 +354,16 @@ def create_or_update_datastore_configuration_parameter(name, data_type=data_type, max_size=max_size, min_size=min_size, - deleted=False, ) get_db_api().save(config) -def load_datastore_configuration_parameters(datastore, - datastore_version, - config_file): +def load_datastore_configuration_parameters(datastore, datastore_version, + config_file, version_number=None): get_db_api().configure_db(CONF) (ds, ds_v) = dstore_models.get_datastore_version( - type=datastore, version=datastore_version, return_inactive=True) + type=datastore, version=datastore_version, return_inactive=True, + version_number=version_number) with open(config_file) as f: config = json.load(f) for param in config['configuration-parameters']: @@ -378,10 +377,12 @@ def load_datastore_configuration_parameters(datastore, ) -def remove_datastore_configuration_parameters(datastore, datastore_version): +def remove_datastore_configuration_parameters(datastore, datastore_version, + version_number=None): get_db_api().configure_db(CONF) (ds, ds_version) = dstore_models.get_datastore_version( - type=datastore, version=datastore_version, return_inactive=True) + type=datastore, version=datastore_version, return_inactive=True, + version_number=version_number) db_params = DatastoreConfigurationParameters.load_parameters(ds_version.id) for db_param in db_params: db_param.delete() diff --git a/trove/configuration/views.py b/trove/configuration/views.py index d9b09c19..3984a729 100644 --- a/trove/configuration/views.py +++ b/trove/configuration/views.py @@ -99,12 +99,14 @@ class DetailedConfigurationView(object): "created": self.configuration.created, "updated": self.configuration.updated, "instance_count": - getattr(self.configuration, "instance_count", 0), + getattr(self.configuration, "instance_count", 0), "datastore_name": self.configuration.datastore.name, "datastore_version_id": - self.configuration.datastore_version_id, + self.configuration.datastore_version_id, "datastore_version_name": - self.configuration.datastore_version.name + self.configuration.datastore_version.name, + "datastore_version_number": + self.configuration.datastore_version.version } return {"configuration": configuration_dict} diff --git a/trove/datastore/models.py b/trove/datastore/models.py index bf989133..f63136f1 100644 --- a/trove/datastore/models.py +++ b/trove/datastore/models.py @@ -16,6 +16,7 @@ # under the License. from oslo_log import log as logging +from oslo_utils import uuidutils from trove.common import cfg from trove.common.clients import create_nova_client @@ -63,7 +64,7 @@ class DBCapabilityOverrides(dbmodels.DatabaseModelBase): class DBDatastoreVersion(dbmodels.DatabaseModelBase): _data_fields = ['datastore_id', 'name', 'image_id', 'image_tags', - 'packages', 'active', 'manager'] + 'packages', 'active', 'manager', 'version'] _table_name = 'datastore_versions' @@ -399,18 +400,30 @@ class DatastoreVersion(object): return "%s(%s)" % (self.name, self.id) @classmethod - def load(cls, datastore, id_or_name): - try: + def load(cls, datastore, id_or_name, version=None): + if uuidutils.is_uuid_like(id_or_name): return cls(DBDatastoreVersion.find_by(datastore_id=datastore.id, id=id_or_name)) - except exception.ModelNotFoundError: + + if not version: versions = DBDatastoreVersion.find_all(datastore_id=datastore.id, name=id_or_name) if versions.count() == 0: raise exception.DatastoreVersionNotFound(version=id_or_name) if versions.count() > 1: - raise exception.NoUniqueMatch(name=id_or_name) - return cls(versions.first()) + raise exception.DatastoreVersionsNoUniqueMatch(name=id_or_name) + + db_version = versions.first() + else: + try: + db_version = DBDatastoreVersion.find_by( + datastore_id=datastore.id, + name=id_or_name, + version=version) + except exception.ModelNotFoundError: + raise exception.DatastoreVersionNotFound(version=version) + + return cls(db_version) @classmethod def load_by_uuid(cls, uuid): @@ -474,6 +487,10 @@ class DatastoreVersion(object): return self._capabilities + @property + def version(self): + return self.db_info.version + class DatastoreVersions(object): @@ -501,7 +518,8 @@ class DatastoreVersions(object): yield item -def get_datastore_version(type=None, version=None, return_inactive=False): +def get_datastore_version(type=None, version=None, return_inactive=False, + version_number=None): datastore = type or CONF.default_datastore if not datastore: raise exception.DatastoreDefaultDatastoreNotDefined() @@ -513,11 +531,12 @@ def get_datastore_version(type=None, version=None, return_inactive=False): datastore=datastore) raise - version = version or datastore.default_version_id - if not version: + version_id = version or datastore.default_version_id + if not version_id: raise exception.DatastoreDefaultVersionNotFound( datastore=datastore.name) - datastore_version = DatastoreVersion.load(datastore, version) + datastore_version = DatastoreVersion.load(datastore, version_id, + version=version_number) if datastore_version.datastore_id != datastore.id: raise exception.DatastoreNoVersion(datastore=datastore.name, version=datastore_version.name) @@ -581,32 +600,36 @@ def update_datastore(name, default_version): def update_datastore_version(datastore, name, manager, image_id, image_tags, - packages, active): + packages, active, version=None, new_name=None): + """Create or update datastore version.""" + version = version or name db_api.configure_db(CONF) datastore = Datastore.load(datastore) try: - version = DBDatastoreVersion.find_by(datastore_id=datastore.id, - name=name) + ds_version = DBDatastoreVersion.find_by(datastore_id=datastore.id, + name=name, + version=version) except exception.ModelNotFoundError: # Create a new one - version = DBDatastoreVersion() - version.id = utils.generate_uuid() - version.name = name - version.datastore_id = datastore.id - version.manager = manager - version.image_id = image_id - version.image_tags = (",".join(image_tags) - if type(image_tags) is list else image_tags) - version.packages = packages - version.active = active + ds_version = DBDatastoreVersion() + ds_version.id = utils.generate_uuid() + ds_version.version = version + ds_version.datastore_id = datastore.id + ds_version.name = new_name or name + ds_version.manager = manager + ds_version.image_id = image_id + ds_version.image_tags = (",".join(image_tags) + if type(image_tags) is list else image_tags) + ds_version.packages = packages + ds_version.active = active - db_api.save(version) + db_api.save(ds_version) class DatastoreVersionMetadata(object): @classmethod - def _datastore_version_find(cls, datastore_name, - datastore_version_name): + def datastore_version_find(cls, datastore_name, + datastore_version_name, version_number=None): """ Helper to find a datastore version id for a given datastore and datastore version name. @@ -615,17 +638,31 @@ class DatastoreVersionMetadata(object): db_ds_record = DBDatastore.find_by( name=datastore_name ) - db_dsv_record = DBDatastoreVersion.find_by( - datastore_id=db_ds_record.id, - name=datastore_version_name - ) + + if not version_number: + db_dsv_records = DBDatastoreVersion.find_all( + datastore_id=db_ds_record.id, + name=datastore_version_name, + ) + if db_dsv_records.count() == 0: + raise exception.DatastoreVersionNotFound( + version=datastore_version_name) + if db_dsv_records.count() > 1: + raise exception.DatastoreVersionsNoUniqueMatch( + name=datastore_version_name) + + db_dsv_record = db_dsv_records.first() + else: + db_dsv_record = DBDatastoreVersion.find_by( + datastore_id=db_ds_record.id, + name=datastore_version_name, + version=version_number + ) return db_dsv_record.id @classmethod - def _datastore_version_metadata_add(cls, datastore_name, - datastore_version_name, - datastore_version_id, + def _datastore_version_metadata_add(cls, datastore_version_id, key, value, exception_class): """ Create a record of the specified key and value in the @@ -646,8 +683,7 @@ class DatastoreVersionMetadata(object): return else: raise exception_class( - datastore=datastore_name, - datastore_version=datastore_version_name, + datastore_version_id=datastore_version_id, id=value) except exception.NotFound: pass @@ -658,8 +694,7 @@ class DatastoreVersionMetadata(object): key=key, value=value) @classmethod - def _datastore_version_metadata_delete(cls, datastore_name, - datastore_version_name, + def _datastore_version_metadata_delete(cls, datastore_version_id, key, value, exception_class): """ Delete a record of the specified key and value in the @@ -668,11 +703,6 @@ class DatastoreVersionMetadata(object): # if an association does not exist, raise an exception # if a deleted association exists, raise an exception # if an un-deleted association exists, delete it - - datastore_version_id = cls._datastore_version_find( - datastore_name, - datastore_version_name) - try: db_record = DBDatastoreVersionMetadata.find_by( datastore_version_id=datastore_version_id, @@ -682,96 +712,69 @@ class DatastoreVersionMetadata(object): return else: raise exception_class( - datastore=datastore_name, - datastore_version=datastore_version_name, + datastore_version_id=datastore_version_id, id=value) except exception.ModelNotFoundError: - raise exception_class(datastore=datastore_name, - datastore_version=datastore_version_name, + raise exception_class(datastore_version_id=datastore_version_id, id=value) @classmethod - def add_datastore_version_flavor_association(cls, datastore_name, - datastore_version_name, + def add_datastore_version_flavor_association(cls, datastore_version_id, flavor_ids): - datastore_version_id = cls._datastore_version_find( - datastore_name, - datastore_version_name) - for flavor_id in flavor_ids: cls._datastore_version_metadata_add( - datastore_name, datastore_version_name, datastore_version_id, 'flavor', flavor_id, exception.DatastoreFlavorAssociationAlreadyExists) @classmethod - def delete_datastore_version_flavor_association(cls, datastore_name, - datastore_version_name, + def delete_datastore_version_flavor_association(cls, datastore_version_id, flavor_id): cls._datastore_version_metadata_delete( - datastore_name, datastore_version_name, 'flavor', flavor_id, + datastore_version_id, 'flavor', flavor_id, exception.DatastoreFlavorAssociationNotFound) @classmethod def list_datastore_version_flavor_associations(cls, context, - datastore_type, datastore_version_id): - if datastore_type and datastore_version_id: - """ - All nova flavors are permitted for a datastore_version unless - one or more entries are found in datastore_version_metadata, - in which case only those are permitted. - """ - (datastore, datastore_version) = get_datastore_version( - type=datastore_type, version=datastore_version_id) - # If datastore_version_id and flavor key exists in the - # metadata table return all the associated flavors for - # that datastore version. - nova_flavors = create_nova_client(context).flavors.list() - bound_flavors = DBDatastoreVersionMetadata.find_all( - datastore_version_id=datastore_version.id, - key='flavor', deleted=False - ) - if (bound_flavors.count() != 0): - bound_flavors = tuple(f.value for f in bound_flavors) - # Generate a filtered list of nova flavors - ds_nova_flavors = (f for f in nova_flavors - if f.id in bound_flavors) - associated_flavors = tuple(flavor_model(flavor=item) - for item in ds_nova_flavors) - else: - # Return all nova flavors if no flavor metadata found - # for datastore_version. - associated_flavors = tuple(flavor_model(flavor=item) - for item in nova_flavors) - return associated_flavors + """Get allowed flavors for a given datastore version. + + All nova flavors are permitted for a datastore_version unless + one or more entries are found in datastore_version_metadata, + in which case only those are permitted. + """ + nova_flavors = create_nova_client(context).flavors.list() + bound_flavors = DBDatastoreVersionMetadata.find_all( + datastore_version_id=datastore_version_id, + key='flavor', deleted=False + ) + if (bound_flavors.count() != 0): + bound_flavors = tuple(f.value for f in bound_flavors) + # Generate a filtered list of nova flavors + ds_nova_flavors = (f for f in nova_flavors + if f.id in bound_flavors) + associated_flavors = tuple(flavor_model(flavor=item) + for item in ds_nova_flavors) else: - msg = _("Specify both the datastore and datastore_version_id.") - raise exception.BadRequest(msg) + # Return all nova flavors if no flavor metadata found + # for datastore_version. + associated_flavors = tuple(flavor_model(flavor=item) + for item in nova_flavors) + return associated_flavors @classmethod - def add_datastore_version_volume_type_association(cls, datastore_name, - datastore_version_name, + def add_datastore_version_volume_type_association(cls, + datastore_version_id, volume_type_names): - datastore_version_id = cls._datastore_version_find( - datastore_name, - datastore_version_name) - - # the database record will contain - # datastore_version_id, 'volume_type', volume_type_name for volume_type_name in volume_type_names: cls._datastore_version_metadata_add( - datastore_name, datastore_version_name, datastore_version_id, 'volume_type', volume_type_name, exception.DatastoreVolumeTypeAssociationAlreadyExists) @classmethod def delete_datastore_version_volume_type_association( - cls, datastore_name, - datastore_version_name, - volume_type_name): + cls, datastore_version_id, volume_type_name): cls._datastore_version_metadata_delete( - datastore_name, datastore_version_name, 'volume_type', + datastore_version_id, 'volume_type', volume_type_name, exception.DatastoreVolumeTypeAssociationNotFound) @@ -800,7 +803,7 @@ class DatastoreVersionMetadata(object): List the datastore associations for a given datastore and version. """ if datastore_name and datastore_version_name: - datastore_version_id = cls._datastore_version_find( + datastore_version_id = cls.datastore_version_find( datastore_name, datastore_version_name) return cls.list_datastore_version_volume_type_associations( datastore_version_id) @@ -809,17 +812,13 @@ class DatastoreVersionMetadata(object): raise exception.BadRequest(msg) @classmethod - def datastore_volume_type_associations_exist(cls, - datastore_name, - datastore_version_name): - return cls.list_datastore_volume_type_associations( - datastore_name, - datastore_version_name).count() > 0 + def datastore_volume_type_associations_exist(cls, datastore_version_id): + return cls.list_datastore_version_volume_type_associations( + datastore_version_id).count() > 0 @classmethod def allowed_datastore_version_volume_types(cls, context, - datastore_name, - datastore_version_name): + datastore_version_id): """ List all allowed volume types for a given datastore and datastore version. If datastore version metadata is @@ -827,59 +826,44 @@ class DatastoreVersionMetadata(object): allowed. If datastore version metadata is not provided then all volume types known to cinder are allowed. """ - if datastore_name and datastore_version_name: - # first obtain the list in the dsvmetadata - datastore_version_id = cls._datastore_version_find( - datastore_name, datastore_version_name) - - metadata = cls.list_datastore_version_volume_type_associations( - datastore_version_id) + metadata = cls.list_datastore_version_volume_type_associations( + datastore_version_id) - # then get the list of all volume types - all_volume_types = volume_type_models.VolumeTypes(context) + # then get the list of all volume types + all_volume_types = volume_type_models.VolumeTypes(context) - # if there's metadata: intersect, - # else, whatever cinder has. - if (metadata.count() != 0): - # the volume types from metadata first - ds_volume_types = tuple(f.value for f in metadata) - - # Cinder volume type names are unique, intersect - allowed_volume_types = tuple( - f for f in all_volume_types - if ((f.name in ds_volume_types) or - (f.id in ds_volume_types))) - else: - allowed_volume_types = tuple(all_volume_types) + # if there's metadata: intersect, + # else, whatever cinder has. + if (metadata.count() != 0): + # the volume types from metadata first + ds_volume_types = tuple(f.value for f in metadata) - return allowed_volume_types + # Cinder volume type names are unique, intersect + allowed_volume_types = tuple( + f for f in all_volume_types + if ((f.name in ds_volume_types) or + (f.id in ds_volume_types))) else: - msg = _("Specify the datastore_name and datastore_version_name.") - raise exception.BadRequest(msg) + allowed_volume_types = tuple(all_volume_types) + + return allowed_volume_types @classmethod - def validate_volume_type(cls, context, volume_type, - datastore_name, datastore_version_name): - if cls.datastore_volume_type_associations_exist( - datastore_name, datastore_version_name): + def validate_volume_type(cls, context, volume_type, datastore_version_id): + if cls.datastore_volume_type_associations_exist(datastore_version_id): allowed = cls.allowed_datastore_version_volume_types( - context, datastore_name, datastore_version_name) + context, datastore_version_id) if len(allowed) == 0: raise exception.DatastoreVersionNoVolumeTypes( - datastore=datastore_name, - datastore_version=datastore_version_name) + datastore_version_id=datastore_version_id) if volume_type is None: raise exception.DataStoreVersionVolumeTypeRequired( - datastore=datastore_name, - datastore_version=datastore_version_name) + datastore_version_id=datastore_version_id) allowed_names = tuple(f.name for f in allowed) - for n in allowed_names: - LOG.debug("Volume Type: %s is allowed for datastore " - "%s, version %s." % - (n, datastore_name, datastore_version_name)) + LOG.debug(f"Allowed volume types: {allowed_names}") + if volume_type not in allowed_names: raise exception.DatastoreVolumeTypeAssociationNotFound( - datastore=datastore_name, - version_id=datastore_version_name, + datastore_version_id=datastore_version_id, id=volume_type) diff --git a/trove/datastore/service.py b/trove/datastore/service.py index b9366690..ada7b4ad 100644 --- a/trove/datastore/service.py +++ b/trove/datastore/service.py @@ -93,7 +93,7 @@ class DatastoreController(wsgi.Controller): context = req.environ[wsgi.CONTEXT_KEY] flavors = (models.DatastoreVersionMetadata. list_datastore_version_flavor_associations( - context, datastore, version_id)) + context, version_id)) return wsgi.Result(flavor_views.FlavorsView(flavors, req).data(), 200) def list_associated_volume_types(self, req, tenant_id, datastore, @@ -106,7 +106,7 @@ class DatastoreController(wsgi.Controller): context = req.environ[wsgi.CONTEXT_KEY] volume_types = (models.DatastoreVersionMetadata. allowed_datastore_version_volume_types( - context, datastore, version_id)) + context, version_id)) return wsgi.Result(volume_type_view.VolumeTypesView( volume_types, req).data(), 200) diff --git a/trove/datastore/views.py b/trove/datastore/views.py index 3dfb321a..be607257 100644 --- a/trove/datastore/views.py +++ b/trove/datastore/views.py @@ -80,6 +80,7 @@ class DatastoreVersionView(object): datastore_version_dict = { "id": self.datastore_version.id, "name": self.datastore_version.name, + "version": self.datastore_version.version, "links": self._build_links(), } if include_datastore_id: diff --git a/trove/db/sqlalchemy/migrate_repo/versions/044_remove_datastore_configuration_parameters_deleted.py b/trove/db/sqlalchemy/migrate_repo/versions/044_remove_datastore_configuration_parameters_deleted.py index 7ce81d6a..7b34b000 100644 --- a/trove/db/sqlalchemy/migrate_repo/versions/044_remove_datastore_configuration_parameters_deleted.py +++ b/trove/db/sqlalchemy/migrate_repo/versions/044_remove_datastore_configuration_parameters_deleted.py @@ -11,19 +11,20 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import sqlalchemy +from sqlalchemy import schema -from sqlalchemy.schema import MetaData +from trove.db.sqlalchemy.migrate_repo import schema as trove_schema -from trove.db.sqlalchemy.migrate_repo.schema import Table - -meta = MetaData() +meta = schema.MetaData() def upgrade(migrate_engine): meta.bind = migrate_engine - ds_config_param = Table('datastore_configuration_parameters', meta, - autoload=True) + ds_config_param = trove_schema.Table('datastore_configuration_parameters', + meta, + autoload=True) # Remove records with deleted=1 if 'deleted' in ds_config_param.c: @@ -35,3 +36,32 @@ def upgrade(migrate_engine): if migrate_engine.name != "sqlite": ds_config_param.drop_column('deleted') ds_config_param.drop_column('deleted_at') + else: + # It is not possible to remove a column from a table in SQLite. + # SQLite is just for testing, so we re-create the table. + ds_config_param.drop() + meta.clear() + trove_schema.Table('datastore_versions', meta, autoload=True) + new_table = trove_schema.Table( + 'datastore_configuration_parameters', + meta, + schema.Column('id', trove_schema.String(36), + primary_key=True, nullable=False), + schema.Column('name', trove_schema.String(128), + primary_key=True, nullable=False), + schema.Column('datastore_version_id', trove_schema.String(36), + sqlalchemy.ForeignKey("datastore_versions.id"), + primary_key=True, nullable=False), + schema.Column('restart_required', trove_schema.Boolean(), + nullable=False, default=False), + schema.Column('max_size', trove_schema.String(40)), + schema.Column('min_size', trove_schema.String(40)), + schema.Column('data_type', trove_schema.String(128), + nullable=False), + schema.UniqueConstraint( + 'datastore_version_id', 'name', + name=('UQ_datastore_configuration_parameters_datastore_' + 'version_id_name') + ) + ) + trove_schema.create_tables([new_table]) diff --git a/trove/db/sqlalchemy/migrate_repo/versions/048_add_version_to_datastore_version.py b/trove/db/sqlalchemy/migrate_repo/versions/048_add_version_to_datastore_version.py new file mode 100644 index 00000000..bf0dd268 --- /dev/null +++ b/trove/db/sqlalchemy/migrate_repo/versions/048_add_version_to_datastore_version.py @@ -0,0 +1,71 @@ +# Copyright 2020 Catalyst Cloud +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from migrate.changeset.constraint import UniqueConstraint +from sqlalchemy import text +from sqlalchemy.schema import Column +from sqlalchemy.schema import MetaData +from sqlalchemy.sql.expression import select +from sqlalchemy.sql.expression import update + +from trove.db.sqlalchemy import utils as db_utils +from trove.db.sqlalchemy.migrate_repo.schema import String +from trove.db.sqlalchemy.migrate_repo.schema import Table + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + ds_table = Table('datastores', meta, autoload=True) + ds_version_table = Table('datastore_versions', meta, autoload=True) + ds_version_table.create_column( + Column('version', String(255), nullable=True)) + + ds_versions = select( + columns=[text("id"), text("name")], + from_obj=ds_version_table + ).execute() + + # Use 'name' value as init 'version' value + for version in ds_versions: + update( + table=ds_version_table, + whereclause=text("id='%s'" % version.id), + values=dict(version=version.name) + ).execute() + + # Change unique constraint, need to drop the foreign key first and add back + # later + constraint_names = db_utils.get_foreign_key_constraint_names( + engine=migrate_engine, + table='datastore_versions', + columns=['datastore_id'], + ref_table='datastores', + ref_columns=['id']) + db_utils.drop_foreign_key_constraints( + constraint_names=constraint_names, + columns=[ds_version_table.c.datastore_id], + ref_columns=[ds_table.c.id]) + + UniqueConstraint('datastore_id', 'name', name='ds_versions', + table=ds_version_table).drop() + UniqueConstraint('datastore_id', 'name', 'version', name='ds_versions', + table=ds_version_table).create() + + db_utils.create_foreign_key_constraints( + constraint_names=constraint_names, + columns=[ds_version_table.c.datastore_id], + ref_columns=[ds_table.c.id]) diff --git a/trove/extensions/mgmt/datastores/service.py b/trove/extensions/mgmt/datastores/service.py index fb730c9e..7eeb899e 100644 --- a/trove/extensions/mgmt/datastores/service.py +++ b/trove/extensions/mgmt/datastores/service.py @@ -49,6 +49,9 @@ class DatastoreVersionController(wsgi.Controller): packages = ','.join(packages) active = body['version']['active'] default = body['version'].get('default', False) + # For backward compatibility, use name as default value for version if + # not specified + version_str = body['version'].get('version', version_name) LOG.info("Tenant: '%(tenant)s' is adding the datastore " "version: '%(version)s' to datastore: '%(datastore)s'", @@ -72,12 +75,15 @@ class DatastoreVersionController(wsgi.Controller): datastore.save() try: - models.DatastoreVersion.load(datastore, version_name) - raise exception.DatastoreVersionAlreadyExists(name=version_name) + models.DatastoreVersion.load(datastore, version_name, + version=version_str) + raise exception.DatastoreVersionAlreadyExists( + name=version_name, version=version_str) except exception.DatastoreVersionNotFound: models.update_datastore_version(datastore.name, version_name, manager, image_id, image_tags, - packages, active) + packages, active, + version=version_str) if default: models.update_datastore(datastore.name, version_name) @@ -109,10 +115,11 @@ class DatastoreVersionController(wsgi.Controller): datastore_version = models.DatastoreVersion.load_by_uuid(id) LOG.info("Tenant: '%(tenant)s' is updating the datastore " - "version: '%(version)s' for datastore: '%(datastore)s'", - {'tenant': tenant_id, 'version': datastore_version.name, + "version: '%(id)s' for datastore: '%(datastore)s'", + {'tenant': tenant_id, 'id': id, 'datastore': datastore_version.datastore_name}) + name = body.get('name', datastore_version.name) manager = body.get('datastore_manager', datastore_version.manager) image_id = body.get('image') image_tags = body.get('image_tags') @@ -143,7 +150,9 @@ class DatastoreVersionController(wsgi.Controller): models.update_datastore_version(datastore_version.datastore_name, datastore_version.name, manager, image_id, image_tags, - packages, active) + packages, active, + version=datastore_version.version, + new_name=name) if default: models.update_datastore(datastore_version.datastore_name, @@ -179,6 +188,13 @@ class DatastoreVersionController(wsgi.Controller): {'tenant': tenant_id, 'version': datastore_version.name, 'datastore': datastore.name}) + # Remove the config parameters associated with the datastore version + LOG.debug(f"Deleting config parameters for datastore version {id}") + db_params = config_model.DatastoreConfigurationParameters. \ + load_parameters(id) + for db_param in db_params: + db_param.delete() + if datastore.default_version_id == datastore_version.id: models.update_datastore(datastore.name, None) datastore_version.delete() diff --git a/trove/extensions/mgmt/datastores/views.py b/trove/extensions/mgmt/datastores/views.py index 971bbc0d..ca6328cb 100644 --- a/trove/extensions/mgmt/datastores/views.py +++ b/trove/extensions/mgmt/datastores/views.py @@ -22,6 +22,7 @@ class DatastoreVersionView(object): datastore_version_dict = { "id": self.datastore_version.id, "name": self.datastore_version.name, + "version": self.datastore_version.version, "datastore_id": self.datastore_version.datastore_id, "datastore_name": self.datastore_version.datastore_name, "datastore_manager": self.datastore_version.manager, diff --git a/trove/guestagent/datastore/mariadb/service.py b/trove/guestagent/datastore/mariadb/service.py index 1edb0122..891e37fb 100644 --- a/trove/guestagent/datastore/mariadb/service.py +++ b/trove/guestagent/datastore/mariadb/service.py @@ -57,13 +57,17 @@ class MariaDBApp(mysql_service.BaseMySqlApp): with mysql_util.SqlClient(self.get_engine()) as client: return client.execute('SELECT @@global.gtid_binlog_pos').first()[0] + def _get_gtid_slave_executed(self): + with mysql_util.SqlClient(self.get_engine()) as client: + return client.execute('SELECT @@global.gtid_slave_pos').first()[0] + def get_last_txn(self): master_UUID = self._get_master_UUID() last_txn_id = '0' - gtid_executed = self._get_gtid_executed() + gtid_executed = self._get_gtid_slave_executed() for gtid_set in gtid_executed.split(','): uuid_set = gtid_set.split('-') - if uuid_set[1] == master_UUID: + if str(uuid_set[1]) == str(master_UUID): last_txn_id = uuid_set[-1] break return master_UUID, int(last_txn_id) diff --git a/trove/guestagent/datastore/mysql/service.py b/trove/guestagent/datastore/mysql/service.py index 891996ce..810e17ae 100644 --- a/trove/guestagent/datastore/mysql/service.py +++ b/trove/guestagent/datastore/mysql/service.py @@ -50,7 +50,7 @@ class MySqlApp(service.BaseMySqlApp): gtid_executed = self._get_gtid_executed() for gtid_set in gtid_executed.split(','): uuid_set = gtid_set.split(':') - if uuid_set[0] == master_UUID: + if str(uuid_set[0]) == str(master_UUID): last_txn_id = uuid_set[-1].split('-')[-1] break return master_UUID, int(last_txn_id) diff --git a/trove/guestagent/strategies/replication/mariadb_gtid.py b/trove/guestagent/strategies/replication/mariadb_gtid.py index b95853bb..4909ee66 100644 --- a/trove/guestagent/strategies/replication/mariadb_gtid.py +++ b/trove/guestagent/strategies/replication/mariadb_gtid.py @@ -42,15 +42,11 @@ class MariaDBGTIDReplication(mysql_base.MysqlReplicationBase): logging_config = master_info['log_position'] last_gtid = '' - if 'gtid_pos' in logging_config: - # This will happen during master failover. - last_gtid = logging_config['gtid_pos'] - elif 'dataset' in master_info: + if 'dataset' in master_info: # This will happen when initial replication is set up. last_gtid = self.read_last_master_gtid(service) - - set_gtid_cmd = "SET GLOBAL gtid_slave_pos='%s';" % last_gtid - service.execute_sql(set_gtid_cmd) + set_gtid_cmd = "SET GLOBAL gtid_slave_pos='%s';" % last_gtid + service.execute_sql(set_gtid_cmd) change_master_cmd = ( "CHANGE MASTER TO " diff --git a/trove/instance/models.py b/trove/instance/models.py index 4d9396a6..1c3079ea 100644 --- a/trove/instance/models.py +++ b/trove/instance/models.py @@ -716,8 +716,8 @@ class BaseInstance(SimpleInstance): self.update_db(task_status=InstanceTasks.DELETING, configuration_id=None) task_api.API(self.context).delete_instance(self.id) - - deltas = {'instances': -1} + flavor = self.get_flavor() + deltas = {'instances': -1, 'ram': -flavor.ram} if self.volume_support: deltas['volumes'] = -self.volume_size return run_with_quotas(self.tenant_id, @@ -816,12 +816,8 @@ class BaseInstance(SimpleInstance): def server_is_finished(): try: server = self.nova_client.servers.get(self.server_id) - if not self.server_status_matches(['SHUTDOWN', 'ACTIVE'], - server=server): - LOG.warning("Server %(vm_id)s entered ERROR status " - "when deleting instance %(instance_id)s!", - {'vm_id': self.server_id, - 'instance_id': self.id}) + LOG.debug(f"Compute server {self.server_id} status " + f"{server.status}") return False except nova_exceptions.NotFound: return True @@ -837,17 +833,17 @@ class BaseInstance(SimpleInstance): "Timeout deleting compute server %(vm_id)s", {'instance_id': self.id, 'vm_id': self.server_id}) - # If volume has been resized it must be manually removed - try: - if self.volume_id: - volume = self.volume_client.volumes.get(self.volume_id) - if volume.status in ["available", "error"]: - LOG.info("Deleting volume %s for instance %s", - self.volume_id, self.id) - volume.delete() - except Exception as e: - LOG.warning("Failed to delete volume for instance %s, error: %s", - self.id, str(e)) + # Cinder volume. + vols = self.volume_client.volumes.list( + search_opts={'name': f'trove-{self.id}'}) + for vol in vols: + LOG.info(f"Deleting volume {vol.id} for instance {self.id}") + + try: + vol.delete() + except Exception as e: + LOG.warning(f"Failed to delete volume {vol.id}({vol.status}) " + f"for instance {self.id}, error: {str(e)}") notification.TroveInstanceDelete( instance=self, @@ -913,6 +909,9 @@ class BaseInstance(SimpleInstance): except exception.ModelNotFoundError: pass + def get_flavor(self): + return self.nova_client.flavors.get(self.flavor_id) + @property def volume_client(self): if not self._volume_client: @@ -1134,9 +1133,8 @@ class Instance(BuiltInstance): valid_flavors = tuple(f.value for f in bound_flavors) if flavor_id not in valid_flavors: raise exception.DatastoreFlavorAssociationNotFound( - datastore=datastore.name, - datastore_version=datastore_version.name, - flavor_id=flavor_id) + datastore_version_id=datastore_version.id, + id=flavor_id) try: flavor = nova_client.flavors.get(flavor_id) except nova_exceptions.NotFound: @@ -1153,7 +1151,7 @@ class Instance(BuiltInstance): cls._validate_remote_datastore(context, region_name, flavor, datastore, datastore_version) - deltas = {'instances': 1} + deltas = {'instances': 1, 'ram': flavor.ram} if volume_support: if replica_source: try: @@ -1167,7 +1165,7 @@ class Instance(BuiltInstance): volume_size = volume.size dvm.validate_volume_type(context, volume_type, - datastore.name, datastore_version.name) + datastore_version.id) validate_volume_size(volume_size) call_args['volume_type'] = volume_type call_args['volume_size'] = volume_size @@ -1336,7 +1334,7 @@ class Instance(BuiltInstance): nics, overrides, slave_of_id, cluster_config, volume_type=volume_type, modules=module_list, locality=locality, access=access, - ds_version=datastore_version.name) + ds_version=datastore_version.version) return SimpleInstance(context, db_info, service_status, root_password, locality=locality) @@ -1351,9 +1349,6 @@ class Instance(BuiltInstance): module_models.InstanceModule.create( context, instance_id, module.id, module.md5) - def get_flavor(self): - return self.nova_client.flavors.get(self.flavor_id) - def get_default_configuration_template(self): flavor = self.get_flavor() LOG.debug("Getting default config template for datastore version " @@ -1371,13 +1366,13 @@ class Instance(BuiltInstance): if self.db_info.cluster_id is not None: raise exception.ClusterInstanceOperationNotSupported() - # Validate that the old and new flavor IDs are not the same, new flavor - # can be found and has ephemeral/volume support if required by the - # current flavor. + # Validate that the old and new flavor IDs are not the same, new + # flavor can be found and has ephemeral/volume support if required + # by the current flavor. if self.flavor_id == new_flavor_id: - raise exception.BadRequest(_("The new flavor id must be different " - "than the current flavor id of '%s'.") - % self.flavor_id) + raise exception.BadRequest( + _("The new flavor id must be different " + "than the current flavor id of '%s'.") % self.flavor_id) try: new_flavor = self.nova_client.flavors.get(new_flavor_id) except nova_exceptions.NotFound: @@ -1390,13 +1385,20 @@ class Instance(BuiltInstance): elif self.device_path is not None: # ephemeral support enabled if new_flavor.ephemeral == 0: - raise exception.LocalStorageNotSpecified(flavor=new_flavor_id) + raise exception.LocalStorageNotSpecified( + flavor=new_flavor_id) + + def _resize_flavor(): + # Set the task to RESIZING and begin the async call before + # returning. + self.update_db(task_status=InstanceTasks.RESIZING) + LOG.debug("Instance %s set to RESIZING.", self.id) + task_api.API(self.context).resize_flavor(self.id, old_flavor, + new_flavor) - # Set the task to RESIZING and begin the async call before returning. - self.update_db(task_status=InstanceTasks.RESIZING) - LOG.debug("Instance %s set to RESIZING.", self.id) - task_api.API(self.context).resize_flavor(self.id, old_flavor, - new_flavor) + return run_with_quotas(self.tenant_id, + {'ram': new_flavor.ram - old_flavor.ram}, + _resize_flavor) def resize_volume(self, new_size): """Resize instance volume. @@ -1484,7 +1486,6 @@ class Instance(BuiltInstance): task_api.API(self.context).promote_to_replica_source(self.id) def eject_replica_source(self): - self.validate_can_perform_action() LOG.info("Ejecting replica source %s from its replication set.", self.id) diff --git a/trove/instance/views.py b/trove/instance/views.py index e0413c18..96d8c796 100644 --- a/trove/instance/views.py +++ b/trove/instance/views.py @@ -129,6 +129,8 @@ class InstanceDetailView(InstanceView): if self.instance.datastore_version: result['instance']['datastore']['version'] = \ self.instance.datastore_version.name + result['instance']['datastore']['version_number'] = \ + self.instance.datastore_version.version if self.instance.fault: result['instance']['fault'] = self._build_fault_info() diff --git a/trove/quota/models.py b/trove/quota/models.py index e2ae0979..03533fd0 100644 --- a/trove/quota/models.py +++ b/trove/quota/models.py @@ -75,6 +75,7 @@ class Resource(object): """Describe a single resource for quota checking.""" INSTANCES = 'instances' + RAM = 'ram' VOLUMES = 'volumes' BACKUPS = 'backups' diff --git a/trove/quota/quota.py b/trove/quota/quota.py index fd5aa766..56891efe 100644 --- a/trove/quota/quota.py +++ b/trove/quota/quota.py @@ -349,6 +349,7 @@ QUOTAS = QuotaEngine() ''' Define all kind of resources here ''' resources = [Resource(Resource.INSTANCES, 'max_instances_per_tenant'), + Resource(Resource.RAM, 'max_ram_per_tenant'), Resource(Resource.BACKUPS, 'max_backups_per_tenant'), Resource(Resource.VOLUMES, 'max_volumes_per_tenant')] diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py index 786ebbdb..585e6da3 100755 --- a/trove/taskmanager/models.py +++ b/trove/taskmanager/models.py @@ -440,6 +440,14 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): self.reset_task_status() TroveInstanceCreate(instance=self, instance_size=flavor['ram']).notify() + except exception.ComputeInstanceNotFound: + # Check if the instance has been deleted by another request. + instance = DBInstance.find_by(id=self.id) + if (instance.deleted or + instance.task_status == InstanceTasks.DELETING): + LOG.warning(f"Instance {self.id} has been deleted during " + f"waiting for creation") + return except (TroveError, PollTimeOut) as ex: LOG.error("Failed to create instance %s, error: %s.", self.id, str(ex)) @@ -786,11 +794,15 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): try: server = self.nova_client.servers.get(c_id) except Exception as e: - raise TroveError( - _("Failed to get server %(server)s for instance %(instance)s, " - "error: %(error)s"), - server=c_id, instance=self.id, error=str(e) - ) + if getattr(e, 'message', '') == 'Not found': + raise exception.ComputeInstanceNotFound(instance_id=self.id, + server_id=c_id) + else: + raise TroveError( + _("Failed to get server %(server)s for instance " + "%(instance)s, error: %(error)s"), + server=c_id, instance=self.id, error=str(e) + ) server_status = server.status if server_status in [InstanceStatus.ERROR, @@ -2100,7 +2112,7 @@ class RebuildAction(ResizeActionBase): LOG.info(f"Sending rebuild request to the instance {self.instance.id}") self.instance.guest.rebuild( - self.instance.datastore_version.name, + self.instance.datastore_version.version, config_contents=config_contents, config_overrides=overrides) LOG.info(f"Waiting for instance {self.instance.id} healthy") diff --git a/trove/tests/api/datastores.py b/trove/tests/api/datastores.py index af8197e5..fe911f63 100644 --- a/trove/tests/api/datastores.py +++ b/trove/tests/api/datastores.py @@ -161,13 +161,9 @@ class DatastoreVersions(object): @test def test_datastore_version_not_found(self): - try: - assert_raises(exceptions.NotFound, - self.rd_client.datastore_versions.get, - self.datastore_active.name, NAME) - except exceptions.BadRequest as e: - assert_equal(e.message, - "Datastore version '%s' cannot be found." % NAME) + assert_raises(exceptions.BadRequest, + self.rd_client.datastore_versions.get, + self.datastore_active.name, NAME) @test def test_datastore_version_list_by_uuid(self): diff --git a/trove/tests/api/instances.py b/trove/tests/api/instances.py index c36b692e..643e120e 100644 --- a/trove/tests/api/instances.py +++ b/trove/tests/api/instances.py @@ -204,7 +204,7 @@ class CheckInstance(AttrCheck): if 'datastore' not in self.instance: self.fail("'datastore' not found in instance.") else: - allowed_attrs = ['type', 'version'] + allowed_attrs = ['type', 'version', 'version_number'] self.contains_allowed_attrs( self.instance['datastore'], allowed_attrs, msg="datastore") @@ -714,18 +714,13 @@ class CreateInstanceFail(object): users = [] datastore = CONFIG.dbaas_datastore datastore_version = "nonexistent" - try: - assert_raises(exceptions.NotFound, - dbaas.instances.create, instance_name, - instance_info.dbaas_flavor_href, - volume, databases, users, - datastore=datastore, - datastore_version=datastore_version, - nics=instance_info.nics) - except exceptions.BadRequest as e: - assert_equal(e.message, - "Datastore version '%s' cannot be found." % - datastore_version) + assert_raises(exceptions.BadRequest, + dbaas.instances.create, instance_name, + instance_info.dbaas_flavor_href, + volume, databases, users, + datastore=datastore, + datastore_version=datastore_version, + nics=instance_info.nics) @test( diff --git a/trove/tests/api/limits.py b/trove/tests/api/limits.py index 2279bdb3..217d7e01 100644 --- a/trove/tests/api/limits.py +++ b/trove/tests/api/limits.py @@ -39,6 +39,7 @@ DEFAULT_RATE = CONF.http_get_rate DEFAULT_MAX_VOLUMES = CONF.max_volumes_per_tenant DEFAULT_MAX_INSTANCES = CONF.max_instances_per_tenant DEFAULT_MAX_BACKUPS = CONF.max_backups_per_tenant +DEFAULT_MAX_RAM = CONF.max_ram_per_tenant def ensure_limits_are_not_faked(func): @@ -109,6 +110,7 @@ class Limits(object): assert_equal(int(abs_limits.max_instances), DEFAULT_MAX_INSTANCES) assert_equal(int(abs_limits.max_backups), DEFAULT_MAX_BACKUPS) assert_equal(int(abs_limits.max_volumes), DEFAULT_MAX_VOLUMES) + assert_equal(int(abs_limits.max_ram), DEFAULT_MAX_RAM) for k in d: assert_equal(d[k].verb, k) @@ -132,6 +134,7 @@ class Limits(object): assert_equal(int(abs_limits.max_instances), DEFAULT_MAX_INSTANCES) assert_equal(int(abs_limits.max_backups), DEFAULT_MAX_BACKUPS) assert_equal(int(abs_limits.max_volumes), DEFAULT_MAX_VOLUMES) + assert_equal(int(abs_limits.max_ram), DEFAULT_MAX_RAM) assert_equal(get.verb, "GET") assert_equal(get.unit, "MINUTE") assert_true(int(get.remaining) <= DEFAULT_RATE - 5) @@ -163,6 +166,8 @@ class Limits(object): DEFAULT_MAX_BACKUPS) assert_equal(int(abs_limits.max_volumes), DEFAULT_MAX_VOLUMES) + assert_equal(int(abs_limits.max_ram,), + DEFAULT_MAX_RAM) except exceptions.OverLimit: encountered = True diff --git a/trove/tests/unittests/api/common/test_limits.py b/trove/tests/unittests/api/common/test_limits.py index ed79639e..73f0ae7c 100644 --- a/trove/tests/unittests/api/common/test_limits.py +++ b/trove/tests/unittests/api/common/test_limits.py @@ -48,6 +48,7 @@ class BaseLimitTestSuite(trove_testtools.TestCase): self.context = trove_testtools.TroveTestContext(self) self.absolute_limits = {"max_instances": 55, "max_volumes": 100, + "max_ram": 200, "max_backups": 40} @@ -114,6 +115,10 @@ class LimitsControllerTest(BaseLimitTestSuite): resource="instances", hard_limit=100), + "ram": Quota(tenant_id=tenant_id, + resource="ram", + hard_limit=200), + "backups": Quota(tenant_id=tenant_id, resource="backups", hard_limit=40), @@ -135,6 +140,7 @@ class LimitsControllerTest(BaseLimitTestSuite): { 'max_instances': 100, 'max_backups': 40, + 'max_ram': 200, 'verb': 'ABSOLUTE', 'max_volumes': 55 }, @@ -798,7 +804,7 @@ class LimitsViewsTest(trove_testtools.TestCase): "resetTime": 1311272226 } ] - abs_view = {"instances": 55, "volumes": 100, "backups": 40} + abs_view = {"instances": 55, "volumes": 100, "backups": 40, 'ram': 200} view_data = views.LimitViews(abs_view, rate_limits) self.assertIsNotNone(view_data) @@ -806,6 +812,7 @@ class LimitsViewsTest(trove_testtools.TestCase): data = view_data.data() expected = {'limits': [{'max_instances': 55, 'max_backups': 40, + 'max_ram': 200, 'verb': 'ABSOLUTE', 'max_volumes': 100}, {'regex': '.*', diff --git a/trove/tests/unittests/configuration/test_service.py b/trove/tests/unittests/configuration/test_service.py new file mode 100644 index 00000000..55bd905c --- /dev/null +++ b/trove/tests/unittests/configuration/test_service.py @@ -0,0 +1,83 @@ +# Copyright 2020 Catalyst Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from unittest import mock + +from trove.common import cfg +from trove.common import wsgi +from trove.configuration import models as config_models +from trove.configuration import service +from trove.datastore import models as ds_models +from trove.tests.unittests import trove_testtools +from trove.tests.unittests.util import util + +CONF = cfg.CONF + + +class TestConfigurationsController(trove_testtools.TestCase): + @classmethod + def setUpClass(cls): + util.init_db() + + cls.ds_name = cls.random_name( + 'datastore', prefix='TestConfigurationsController') + ds_models.update_datastore(name=cls.ds_name, default_version=None) + cls.ds = ds_models.Datastore.load(cls.ds_name) + + ds_version_name = cls.random_name( + 'version', prefix='TestConfigurationsController') + ds_models.update_datastore_version( + cls.ds_name, ds_version_name, 'mysql', '', + ['trove'], '', 1, version='5.7.29') + cls.ds_version = ds_models.DatastoreVersion.load( + cls.ds, ds_version_name, version='5.7.29') + + cls.tenant_id = cls.random_uuid() + cls.config = config_models.Configuration.create( + cls.random_name('configuration'), + '', cls.tenant_id, None, + cls.ds_version.id) + cls.config_id = cls.config.id + + cls.controller = service.ConfigurationsController() + + super(TestConfigurationsController, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + util.cleanup_db() + + super(TestConfigurationsController, cls).tearDownClass() + + def test_show(self): + req_mock = mock.MagicMock( + environ={ + wsgi.CONTEXT_KEY: mock.MagicMock(project_id=self.tenant_id) + } + ) + result = self.controller.show(req_mock, self.tenant_id, + self.config_id) + data = result.data(None).get('configuration') + + expected = { + "id": self.config_id, + "name": self.config.name, + "description": '', + "instance_count": 0, + "datastore_name": self.ds_name, + "datastore_version_id": self.ds_version.id, + "datastore_version_name": self.ds_version.name, + "datastore_version_number": self.ds_version.version + } + + self.assertDictContains(data, expected) diff --git a/trove/tests/unittests/datastore/base.py b/trove/tests/unittests/datastore/base.py index bf38c09a..f543b77f 100644 --- a/trove/tests/unittests/datastore/base.py +++ b/trove/tests/unittests/datastore/base.py @@ -41,15 +41,15 @@ class TestDatastoreBase(trove_testtools.TestCase): datastore_models.update_datastore_version( cls.ds_name, cls.ds_version_name, "mysql", "", "", "", True) - DatastoreVersionMetadata.add_datastore_version_flavor_association( - cls.ds_name, cls.ds_version_name, [cls.flavor_id]) - DatastoreVersionMetadata.add_datastore_version_volume_type_association( - cls.ds_name, cls.ds_version_name, [cls.volume_type]) - cls.datastore_version = DatastoreVersion.load(cls.datastore, cls.ds_version_name) cls.test_id = cls.datastore_version.id + DatastoreVersionMetadata.add_datastore_version_flavor_association( + cls.datastore_version.id, [cls.flavor_id]) + DatastoreVersionMetadata.add_datastore_version_volume_type_association( + cls.datastore_version.id, [cls.volume_type]) + cls.cap1 = Capability.create(cls.capability_name, cls.capability_desc, True) cls.cap2 = Capability.create( diff --git a/trove/tests/unittests/datastore/test_datastore_version_metadata.py b/trove/tests/unittests/datastore/test_datastore_version_metadata.py index 2574ef10..964cab0d 100644 --- a/trove/tests/unittests/datastore/test_datastore_version_metadata.py +++ b/trove/tests/unittests/datastore/test_datastore_version_metadata.py @@ -57,48 +57,40 @@ class TestDatastoreVersionMetadata(TestDatastoreBase): def test_add_existing_flavor_associations(self): dsmetadata = datastore_models.DatastoreVersionMetadata - self.assertRaisesRegex( + self.assertRaises( exception.DatastoreFlavorAssociationAlreadyExists, - "Flavor %s is already associated with datastore %s version %s" - % (self.flavor_id, self.ds_name, self.ds_version_name), dsmetadata.add_datastore_version_flavor_association, - self.ds_name, self.ds_version_name, [self.flavor_id]) + self.test_id, [self.flavor_id]) def test_add_existing_volume_type_associations(self): dsmetadata = datastore_models.DatastoreVersionMetadata self.assertRaises( exception.DatastoreVolumeTypeAssociationAlreadyExists, dsmetadata.add_datastore_version_volume_type_association, - self.ds_name, self.ds_version_name, [self.volume_type]) + self.test_id, [self.volume_type]) def test_delete_nonexistent_flavor_mapping(self): dsmeta = datastore_models.DatastoreVersionMetadata - self.assertRaisesRegex( + self.assertRaises( exception.DatastoreFlavorAssociationNotFound, - "Flavor 2 is not supported for datastore %s version %s" - % (self.ds_name, self.ds_version_name), dsmeta.delete_datastore_version_flavor_association, - self.ds_name, self.ds_version_name, flavor_id=2) + self.test_id, flavor_id=2) def test_delete_nonexistent_volume_type_mapping(self): dsmeta = datastore_models.DatastoreVersionMetadata self.assertRaises( exception.DatastoreVolumeTypeAssociationNotFound, dsmeta.delete_datastore_version_volume_type_association, - self.ds_name, self.ds_version_name, + self.test_id, volume_type_name='some random thing') def test_delete_flavor_mapping(self): flavor_id = 2 dsmetadata = datastore_models.DatastoreVersionMetadata dsmetadata.add_datastore_version_flavor_association( - self.ds_name, - self.ds_version_name, - [flavor_id]) + self.test_id, [flavor_id]) dsmetadata.delete_datastore_version_flavor_association( - self.ds_name, - self.ds_version_name, - flavor_id) + self.test_id, flavor_id) datastore = datastore_models.Datastore.load(self.ds_name) ds_version = datastore_models.DatastoreVersion.load( datastore, @@ -108,27 +100,22 @@ class TestDatastoreVersionMetadata(TestDatastoreBase): self.assertTrue(mapping.deleted) # check update dsmetadata.add_datastore_version_flavor_association( - self.ds_name, self.ds_version_name, [flavor_id]) + self.test_id, [flavor_id]) mapping = datastore_models.DBDatastoreVersionMetadata.find_by( datastore_version_id=ds_version.id, value=flavor_id, key='flavor') self.assertFalse(mapping.deleted) # clear the mapping datastore_models.DatastoreVersionMetadata. \ - delete_datastore_version_flavor_association(self.ds_name, - self.ds_version_name, + delete_datastore_version_flavor_association(self.test_id, flavor_id) def test_delete_volume_type_mapping(self): volume_type = 'this is bogus' dsmetadata = datastore_models.DatastoreVersionMetadata dsmetadata.add_datastore_version_volume_type_association( - self.ds_name, - self.ds_version_name, - [volume_type]) + self.test_id, [volume_type]) dsmetadata.delete_datastore_version_volume_type_association( - self.ds_name, - self.ds_version_name, - volume_type) + self.test_id, volume_type) datastore = datastore_models.Datastore.load(self.ds_name) ds_version = datastore_models.DatastoreVersion.load( datastore, @@ -139,19 +126,17 @@ class TestDatastoreVersionMetadata(TestDatastoreBase): self.assertTrue(mapping.deleted) # check update dsmetadata.add_datastore_version_volume_type_association( - self.ds_name, self.ds_version_name, [volume_type]) + self.test_id, [volume_type]) mapping = datastore_models.DBDatastoreVersionMetadata.find_by( datastore_version_id=ds_version.id, value=volume_type, key='volume_type') self.assertFalse(mapping.deleted) # clear the mapping dsmetadata.delete_datastore_version_volume_type_association( - self.ds_name, - self.ds_version_name, - volume_type) + self.test_id, volume_type) @mock.patch.object(datastore_models.DatastoreVersionMetadata, - '_datastore_version_find') + 'datastore_version_find') @mock.patch.object(datastore_models.DatastoreVersionMetadata, 'list_datastore_version_volume_type_associations') @mock.patch.object(clients, 'create_cinder_client') @@ -179,7 +164,7 @@ class TestDatastoreVersionMetadata(TestDatastoreBase): mock_list.return_value = mock_trove_list_result return self.dsmetadata.allowed_datastore_version_volume_types( - None, 'ds', 'dsv') + None, self.random_uuid()) def _assert_equal_types(self, test_dict, output_obj): self.assertEqual(test_dict.get('id'), output_obj.id) diff --git a/trove/tests/unittests/extensions/mgmt/datastores/test_service.py b/trove/tests/unittests/extensions/mgmt/datastores/test_service.py index 185a740e..79af2b07 100644 --- a/trove/tests/unittests/extensions/mgmt/datastores/test_service.py +++ b/trove/tests/unittests/extensions/mgmt/datastores/test_service.py @@ -21,6 +21,7 @@ import jsonschema from trove.common import clients from trove.common import exception +from trove.configuration import models as config_models from trove.datastore import models from trove.extensions.mgmt.datastores.service import DatastoreVersionController from trove.tests.unittests import trove_testtools @@ -32,6 +33,7 @@ class TestDatastoreVersionController(trove_testtools.TestCase): def setUpClass(cls): util.init_db() cls.ds_name = cls.random_name('datastore') + cls.ds_version_number = '5.7.30' models.update_datastore(name=cls.ds_name, default_version=None) models.update_datastore_version( @@ -39,11 +41,12 @@ class TestDatastoreVersionController(trove_testtools.TestCase): 1) models.update_datastore_version( cls.ds_name, 'test_vr2', 'mysql', cls.random_uuid(), '', 'pkg-1', - 1) + 1, version=cls.ds_version_number) cls.ds = models.Datastore.load(cls.ds_name) cls.ds_version1 = models.DatastoreVersion.load(cls.ds, 'test_vr1') - cls.ds_version2 = models.DatastoreVersion.load(cls.ds, 'test_vr2') + cls.ds_version2 = models.DatastoreVersion.load( + cls.ds, 'test_vr2', version=cls.ds_version_number) cls.version_controller = DatastoreVersionController() super(TestDatastoreVersionController, cls).setUpClass() @@ -136,6 +139,34 @@ class TestDatastoreVersionController(trove_testtools.TestCase): new_ver = models.DatastoreVersion.load(self.ds, ver_name) self.assertEqual(image_id, new_ver.image_id) + self.assertEqual(ver_name, new_ver.version) + + @patch.object(clients, 'create_glance_client') + def test_create_same_version_number(self, mock_glance_client): + image_id = self.random_uuid() + ver_name = self.random_name('dsversion') + body = { + "version": { + "datastore_name": self.ds_name, + "name": ver_name, + "datastore_manager": "mysql", + "image": image_id, + "image_tags": [], + "packages": "", + "active": True, + "default": False, + "version": self.ds_version_number + } + } + output = self.version_controller.create(MagicMock(), body, mock.ANY) + self.assertEqual(202, output.status) + + new_ver = models.DatastoreVersion.load(self.ds, ver_name, + version=self.ds_version_number) + self.assertEqual(image_id, new_ver.image_id) + self.assertEqual(ver_name, new_ver.name) + self.assertEqual(self.ds_version_number, new_ver.version) + self.assertNotEqual(self.ds_version2.id, new_ver.id) @patch.object(clients, 'create_glance_client') def test_create_by_image_tags(self, mock_create_client): @@ -224,6 +255,26 @@ class TestDatastoreVersionController(trove_testtools.TestCase): exception.ImageNotFound, self.version_controller.create, MagicMock(), body, mock.ANY) + def test_update_name(self): + new_name = self.random_name('ds-version-name') + body = { + "name": new_name + } + + orig_ver = models.DatastoreVersion.load(self.ds, self.ds_version1.id) + + output = self.version_controller.edit(MagicMock(), body, mock.ANY, + self.ds_version1.id) + self.assertEqual(202, output.status) + + updated_ver = models.DatastoreVersion.load(self.ds, + self.ds_version1.id) + + self.assertEqual(new_name, updated_ver.name) + self.assertEqual(orig_ver.image_id, updated_ver.image_id) + self.assertEqual(orig_ver.image_tags, updated_ver.image_tags) + self.assertEqual(orig_ver.version, updated_ver.version) + @patch.object(clients, 'create_glance_client') def test_update_image(self, mock_create_client): new_image = self.random_uuid() @@ -268,6 +319,12 @@ class TestDatastoreVersionController(trove_testtools.TestCase): self.ds_name, name, 'mysql', self.random_uuid(), '', '', 1) ver = models.DatastoreVersion.load(self.ds, name) + # Add config param for the datastore version. Should be automatically + # removed. + param_name = self.random_name('param') + config_models.create_or_update_datastore_configuration_parameter( + param_name, ver.id, False, 'string', None, None) + output = self.version_controller.delete(MagicMock(), mock.ANY, ver.id) @@ -277,6 +334,12 @@ class TestDatastoreVersionController(trove_testtools.TestCase): exception.DatastoreVersionNotFound, models.DatastoreVersion.load_by_uuid, ver.id) + config_params_cls = config_models.DatastoreConfigurationParameters + self.assertRaises( + exception.NotFound, + config_params_cls.load_parameter_by_name, + ver.id, param_name) + def test_index(self): output = self.version_controller.index(MagicMock(), mock.ANY) self.assertEqual(200, output.status) @@ -304,6 +367,8 @@ class TestDatastoreVersionController(trove_testtools.TestCase): output._data['version']['packages']) self.assertEqual(self.ds_version2.active, output._data['version']['active']) + self.assertEqual(self.ds_version2.version, + output._data['version']['version']) def test_show_image_tags(self): ver_name = self.random_name('dsversion') diff --git a/trove/tests/unittests/instance/test_service.py b/trove/tests/unittests/instance/test_service.py index 07444a4c..a38c62f5 100644 --- a/trove/tests/unittests/instance/test_service.py +++ b/trove/tests/unittests/instance/test_service.py @@ -13,7 +13,9 @@ # limitations under the License. from unittest import mock +from trove.common import cfg from trove.common import clients +from trove.common import exception from trove.datastore import models as ds_models from trove.instance import models as ins_models from trove.instance import service @@ -21,6 +23,8 @@ from trove.instance import service_status as srvstatus from trove.tests.unittests import trove_testtools from trove.tests.unittests.util import util +CONF = cfg.CONF + class TestInstanceController(trove_testtools.TestCase): @classmethod @@ -37,7 +41,14 @@ class TestInstanceController(trove_testtools.TestCase): 1) ds_models.update_datastore_version( cls.ds_name, 'test_image_tags', 'mysql', '', ['trove', 'mysql'], - '', 1) + '', 1, version='test_image_tags version') + ds_models.update_datastore_version( + cls.ds_name, 'test_version', 'mysql', '', ['trove'], '', 1, + version='version 1') + ds_models.update_datastore_version( + cls.ds_name, 'test_version', 'mysql', '', ['trove'], '', 1, + version='version 2') + cls.ds_version_imageid = ds_models.DatastoreVersion.load( cls.ds, 'test_image_id') cls.ds_version_imagetags = ds_models.DatastoreVersion.load( @@ -61,19 +72,22 @@ class TestInstanceController(trove_testtools.TestCase): @mock.patch('trove.instance.models.Instance.create') def test_create_by_ds_version_image_tags(self, mock_model_create, mock_create_client): + image_id = self.random_uuid() mock_glance_client = mock.MagicMock() - mock_glance_client.images.list.return_value = [ - {'id': self.random_uuid()}] + mock_glance_client.images.list.return_value = [{'id': image_id}] mock_create_client.return_value = mock_glance_client + name = self.random_name(name='instance', + prefix='TestInstanceController') + flavor = self.random_uuid() body = { 'instance': { - 'name': self.random_name(name='instance', - prefix='TestInstanceController'), - 'flavorRef': self.random_uuid(), + 'name': name, + 'flavorRef': flavor, 'datastore': { 'type': self.ds_name, - 'version': self.ds_version_imagetags.name + 'version': self.ds_version_imagetags.name, + 'version_number': self.ds_version_imagetags.version } } } @@ -85,6 +99,40 @@ class TestInstanceController(trove_testtools.TestCase): sort='created_at:desc', limit=1 ) + mock_model_create.assert_called_once_with( + mock.ANY, name, flavor, image_id, + [], [], + mock.ANY, mock.ANY, + None, None, None, [], None, None, + replica_count=None, volume_type=None, modules=None, locality=None, + region_name=CONF.service_credentials.region_name, access=None + ) + args = mock_model_create.call_args[0] + actual_ds_version = args[7] + self.assertEqual(self.ds_version_imagetags.name, + actual_ds_version.name) + self.assertEqual(self.ds_version_imagetags.version, + actual_ds_version.version) + + def test_create_multiple_versions(self): + body = { + 'instance': { + 'name': self.random_name(name='instance', + prefix='TestInstanceController'), + 'flavorRef': self.random_uuid(), + 'datastore': { + 'type': self.ds_name, + 'version': 'test_version' + } + } + } + + self.assertRaises( + exception.DatastoreVersionsNoUniqueMatch, + self.controller.create, + mock.MagicMock(), body, mock.ANY + ) + @mock.patch.object(clients, 'create_nova_client', return_value=mock.MagicMock()) @mock.patch('trove.rpc.get_client') diff --git a/trove/tests/unittests/trove_testtools.py b/trove/tests/unittests/trove_testtools.py index 57ada51a..34e67b89 100644 --- a/trove/tests/unittests/trove_testtools.py +++ b/trove/tests/unittests/trove_testtools.py @@ -122,3 +122,11 @@ class TestCase(testtools.TestCase): @classmethod def random_uuid(cls): return str(uuid.uuid4()) + + def assertDictContains(self, parent, child): + """Checks whether child dict is a subset of parent. + + assertDictContainsSubset() in standard Python 2.7 has been deprecated + since Python 3.2 + """ + self.assertEqual(parent, dict(parent, **child)) |