path: root/nova/tests/functional/db/test_instance_mapping.py

#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import mock
from oslo_utils.fixture import uuidsentinel
from oslo_utils import uuidutils

from nova.compute import vm_states
from nova import context
from nova import exception
from nova.objects import cell_mapping
from nova.objects import instance
from nova.objects import instance_mapping
from nova.objects import virtual_interface
from nova import test
from nova.tests import fixtures


sample_mapping = {'instance_uuid': '',
                  'cell_id': 3,
                  'project_id': 'fake-project',
                  'user_id': 'fake-user'}


sample_cell_mapping = {'id': 3,
                       'uuid': '',
                       'name': 'fake-cell',
                       'transport_url': 'rabbit:///',
                       'database_connection': 'mysql:///'}


def create_cell_mapping(**kwargs):
    args = sample_cell_mapping.copy()
    if 'uuid' not in kwargs:
        args['uuid'] = uuidutils.generate_uuid()
    args.update(kwargs)
    ctxt = context.RequestContext('fake-user', 'fake-project')
    return cell_mapping.CellMapping._create_in_db(ctxt, args)


def create_mapping(**kwargs):
    args = sample_mapping.copy()
    if 'instance_uuid' not in kwargs:
        args['instance_uuid'] = uuidutils.generate_uuid()
    args.update(kwargs)
    ctxt = context.RequestContext('fake-user', 'fake-project')
    return instance_mapping.InstanceMapping._create_in_db(ctxt, args)


class InstanceMappingTestCase(test.NoDBTestCase):
    USES_DB_SELF = True

    def setUp(self):
        super(InstanceMappingTestCase, self).setUp()
        self.useFixture(fixtures.Database(database='api'))
        self.context = context.RequestContext('fake-user', 'fake-project')
        self.mapping_obj = instance_mapping.InstanceMapping()

    def test_get_by_instance_uuid(self):
        cell_mapping = create_cell_mapping()
        mapping = create_mapping()
        db_mapping = self.mapping_obj._get_by_instance_uuid_from_db(
                self.context, mapping['instance_uuid'])
        for key in [key for key in self.mapping_obj.fields.keys()
                    if key != 'cell_mapping']:
            self.assertEqual(db_mapping[key], mapping[key])
        self.assertEqual(db_mapping['cell_mapping']['id'], cell_mapping['id'])

    def test_get_by_instance_uuid_not_found(self):
        self.assertRaises(exception.InstanceMappingNotFound,
                self.mapping_obj._get_by_instance_uuid_from_db, self.context,
                uuidutils.generate_uuid())

    def test_save_in_db(self):
        mapping = create_mapping()
        cell_mapping = create_cell_mapping()
        self.mapping_obj._save_in_db(self.context, mapping['instance_uuid'],
                {'cell_id': cell_mapping['id']})
        db_mapping = self.mapping_obj._get_by_instance_uuid_from_db(
                self.context, mapping['instance_uuid'])
        for key in [key for key in self.mapping_obj.fields.keys()
                    if key not in ['cell_id', 'cell_mapping', 'updated_at']]:
            self.assertEqual(db_mapping[key], mapping[key])
        self.assertEqual(db_mapping['cell_id'], cell_mapping['id'])

    def test_destroy_in_db(self):
        mapping = create_mapping()
        self.mapping_obj._get_by_instance_uuid_from_db(self.context,
                mapping['instance_uuid'])
        self.mapping_obj._destroy_in_db(self.context, mapping['instance_uuid'])
        self.assertRaises(exception.InstanceMappingNotFound,
                self.mapping_obj._get_by_instance_uuid_from_db, self.context,
                mapping['instance_uuid'])

    def test_cell_id_nullable(self):
        # Just ensure this doesn't raise
        create_mapping(cell_id=None)

    def test_modify_cell_mapping(self):
        inst_mapping = instance_mapping.InstanceMapping(context=self.context)
        inst_mapping.instance_uuid = uuidutils.generate_uuid()
        inst_mapping.project_id = self.context.project_id
        inst_mapping.cell_mapping = None
        inst_mapping.create()

        c_mapping = cell_mapping.CellMapping(
                self.context,
                uuid=uuidutils.generate_uuid(),
                name="cell0",
                transport_url="none:///",
                database_connection="fake:///")
        c_mapping.create()

        inst_mapping.cell_mapping = c_mapping
        inst_mapping.save()

        result_mapping = instance_mapping.InstanceMapping.get_by_instance_uuid(
                                    self.context, inst_mapping.instance_uuid)

        self.assertEqual(result_mapping.cell_mapping.id,
                         c_mapping.id)

    def test_populate_queued_for_delete(self):
        cells = []
        celldbs = fixtures.CellDatabases()

        # Create two cell databases and map them
        for uuid in (uuidsentinel.cell1, uuidsentinel.cell2):
            cm = cell_mapping.CellMapping(context=self.context, uuid=uuid,
                                          database_connection=uuid,
                                          transport_url='fake://')
            cm.create()
            cells.append(cm)
            celldbs.add_cell_database(uuid)
        self.useFixture(celldbs)

        # Create 5 instances per cell, two deleted, one with matching
        # queued_for_delete in the instance mapping
        for cell in cells:
            for i in range(0, 5):
                # Instance 4 should be SOFT_DELETED
                vm_state = (vm_states.SOFT_DELETED if i == 4
                            else vm_states.ACTIVE)

                # Instance 2 should already be marked as queued_for_delete
                qfd = True if i == 2 else None

                with context.target_cell(self.context, cell) as cctxt:
                    inst = instance.Instance(
                        cctxt,
                        vm_state=vm_state,
                        project_id=self.context.project_id,
                        user_id=self.context.user_id)
                    inst.create()
                    if i in (2, 3):
                        # Instances 2 and 3 are hard-deleted
                        inst.destroy()

                instance_mapping.InstanceMapping._create_in_db(
                    self.context,
                    {'project_id': self.context.project_id,
                     'cell_id': cell.id,
                     'queued_for_delete': qfd,
                     'instance_uuid': inst.uuid})

        done, total = instance_mapping.populate_queued_for_delete(self.context,
                                                                  2)
        # First two needed fixing, and honored the limit
        self.assertEqual(2, done)
        self.assertEqual(2, total)

        done, total = instance_mapping.populate_queued_for_delete(self.context,
                                                                  1000)

        # Last six included two that were already done, and spanned to the
        # next cell
        self.assertEqual(6, done)
        self.assertEqual(6, total)

        mappings = instance_mapping.InstanceMappingList.get_by_project_id(
            self.context, self.context.project_id)

        # Check that we have only the expected number of records with
        # True/False (which implies no NULL records).

        # Six deleted instances
        self.assertEqual(6, len(
            [im for im in mappings if im.queued_for_delete is True]))
        # Four non-deleted instances
        self.assertEqual(4, len(
            [im for im in mappings if im.queued_for_delete is False]))

        # Run it again to make sure we don't query the cell database for
        # instances if we didn't get any un-migrated mappings.
        with mock.patch('nova.objects.InstanceList.get_by_filters',
                        new_callable=mock.NonCallableMock):
            done, total = instance_mapping.populate_queued_for_delete(
                self.context, 1000)
        self.assertEqual(0, done)
        self.assertEqual(0, total)

    def test_user_id_not_set_if_null_from_db(self):
        # Create an instance mapping with user_id=None.
        db_mapping = create_mapping(user_id=None)
        self.assertIsNone(db_mapping['user_id'])
        # Get the mapping to run convert from db object to versioned object.
        im = instance_mapping.InstanceMapping.get_by_instance_uuid(
            self.context, db_mapping['instance_uuid'])
        # Verify the user_id is not set.
        self.assertNotIn('user_id', im)

    @mock.patch('nova.objects.instance_mapping.LOG.warning')
    def test_populate_user_id(self, mock_log_warning):
        cells = []
        celldbs = fixtures.CellDatabases()

        # Create two cell databases and map them
        for uuid in (uuidsentinel.cell1, uuidsentinel.cell2):
            cm = cell_mapping.CellMapping(context=self.context, uuid=uuid,
                                          database_connection=uuid,
                                          transport_url='fake://')
            cm.create()
            cells.append(cm)
            celldbs.add_cell_database(uuid)
        self.useFixture(celldbs)

        # Create 5 instances per cell
        for cell in cells:
            for i in range(0, 5):
                with context.target_cell(self.context, cell) as cctxt:
                    inst = instance.Instance(
                        cctxt,
                        project_id=self.context.project_id,
                        user_id=self.context.user_id)
                    inst.create()
                # Make every other mapping have a NULL user_id
                # Will be a total of four mappings with NULL user_id
                user_id = self.context.user_id if i % 2 == 0 else None
                create_mapping(project_id=self.context.project_id,
                               user_id=user_id, cell_id=cell.id,
                               instance_uuid=inst.uuid)

        # Create a SOFT_DELETED instance with a user_id=None instance mapping.
        # This should get migrated.
        with context.target_cell(self.context, cells[0]) as cctxt:
            inst = instance.Instance(
                cctxt, project_id=self.context.project_id,
                user_id=self.context.user_id, vm_state=vm_states.SOFT_DELETED)
            inst.create()
        create_mapping(project_id=self.context.project_id, user_id=None,
                       cell_id=cells[0].id, instance_uuid=inst.uuid,
                       queued_for_delete=True)

        # Create a deleted instance with a user_id=None instance mapping.
        # This should get migrated.
        with context.target_cell(self.context, cells[1]) as cctxt:
            inst = instance.Instance(
                cctxt, project_id=self.context.project_id,
                user_id=self.context.user_id)
            inst.create()
            inst.destroy()
        create_mapping(project_id=self.context.project_id, user_id=None,
                       cell_id=cells[1].id, instance_uuid=inst.uuid,
                       queued_for_delete=True)

        # Create an instance mapping for an instance not yet scheduled. It
        # should not get migrated because we won't know what user_id to use.
        unscheduled = create_mapping(project_id=self.context.project_id,
                                     user_id=None, cell_id=None)

        # Create two instance mappings for instances that no longer exist.
        # Example: residue from a manual cleanup or after a periodic compute
        # purge and before a database archive. This record should not get
        # migrated.
        nonexistent = []
        for i in range(2):
            nonexistent.append(
                create_mapping(project_id=self.context.project_id,
                               user_id=None, cell_id=cells[i].id,
                               instance_uuid=uuidutils.generate_uuid()))

        # Create an instance mapping simulating a virtual interface migration
        # marker instance which has had map_instances run on it.
        # This should not be found by the migration.
        create_mapping(project_id=virtual_interface.FAKE_UUID, user_id=None)

        found, done = instance_mapping.populate_user_id(self.context, 2)
        # Two needed fixing, and honored the limit.
        self.assertEqual(2, found)
        self.assertEqual(2, done)

        found, done = instance_mapping.populate_user_id(self.context, 1000)
        # Only four left were fixable. The fifth instance found has no
        # cell and cannot be migrated yet. The 6th and 7th instances found have
        # no corresponding instance records and cannot be migrated.
        self.assertEqual(7, found)
        self.assertEqual(4, done)

        # Verify the orphaned instance mappings warning log message was only
        # emitted once.
        mock_log_warning.assert_called_once()

        # Check that we have only the expected number of records with
        # user_id set. We created 10 instances (5 per cell with 2 per cell
        # with NULL user_id), 1 SOFT_DELETED instance with NULL user_id,
        # 1 deleted instance with NULL user_id, and 1 not-yet-scheduled
        # instance with NULL user_id.
        # We expect 12 of them to have user_id set after migration (15 total,
        # with the not-yet-scheduled instance and the orphaned instance
        # mappings ignored).
        ims = instance_mapping.InstanceMappingList.get_by_project_id(
            self.context, self.context.project_id)
        self.assertEqual(12, len(
            [im for im in ims if 'user_id' in im]))

        # Check that one instance mapping record (not yet scheduled) has not
        # been migrated by this script.
        # Check that two other instance mapping records (no longer existing
        # instances) have not been migrated by this script.
        self.assertEqual(15, len(ims))

        # Set the cell and create the instance for the mapping without a cell,
        # then run the migration again.
        unscheduled = instance_mapping.InstanceMapping.get_by_instance_uuid(
            self.context, unscheduled['instance_uuid'])
        unscheduled.cell_mapping = cells[0]
        unscheduled.save()
        with context.target_cell(self.context, cells[0]) as cctxt:
            inst = instance.Instance(
                cctxt,
                uuid=unscheduled.instance_uuid,
                project_id=self.context.project_id,
                user_id=self.context.user_id)
            inst.create()
        found, done = instance_mapping.populate_user_id(self.context, 1000)
        # Should have found the not-yet-scheduled instance and the orphaned
        # instance mappings.
        self.assertEqual(3, found)
        # Should have only migrated the not-yet-schedule instance.
        self.assertEqual(1, done)

        # Delete the orphaned instance mapping (simulate manual cleanup by an
        # operator).
        for db_im in nonexistent:
            nonexist = instance_mapping.InstanceMapping.get_by_instance_uuid(
                self.context, db_im['instance_uuid'])
            nonexist.destroy()

        # Run the script one last time to make sure it finds nothing left to
        # migrate.
        found, done = instance_mapping.populate_user_id(self.context, 1000)
        self.assertEqual(0, found)
        self.assertEqual(0, done)

    @mock.patch('nova.objects.InstanceList.get_by_filters')
    def test_populate_user_id_instance_get_fail(self, mock_inst_get):
        cells = []
        celldbs = fixtures.CellDatabases()

        # Create two cell databases and map them
        for uuid in (uuidsentinel.cell1, uuidsentinel.cell2):
            cm = cell_mapping.CellMapping(context=self.context, uuid=uuid,
                                          database_connection=uuid,
                                          transport_url='fake://')
            cm.create()
            cells.append(cm)
            celldbs.add_cell_database(uuid)
        self.useFixture(celldbs)

        # Create one instance per cell
        for cell in cells:
            with context.target_cell(self.context, cell) as cctxt:
                inst = instance.Instance(
                    cctxt,
                    project_id=self.context.project_id,
                    user_id=self.context.user_id)
                inst.create()
            create_mapping(project_id=self.context.project_id,
                           user_id=None, cell_id=cell.id,
                           instance_uuid=inst.uuid)

        # Simulate the first cell is down/has some error
        mock_inst_get.side_effect = [test.TestingException(),
                                     instance.InstanceList(objects=[inst])]

        found, done = instance_mapping.populate_user_id(self.context, 1000)
        # Verify we continue to the next cell when a down/error cell is
        # encountered.
        self.assertEqual(2, found)
        self.assertEqual(1, done)


class InstanceMappingListTestCase(test.NoDBTestCase):
    USES_DB_SELF = True

    def setUp(self):
        super(InstanceMappingListTestCase, self).setUp()
        self.useFixture(fixtures.Database(database='api'))
        self.context = context.RequestContext('fake-user', 'fake-project')
        self.list_obj = instance_mapping.InstanceMappingList()

    def test_get_by_project_id_from_db(self):
        project_id = 'fake-project'
        mappings = {}
        mapping = create_mapping(project_id=project_id)
        mappings[mapping['instance_uuid']] = mapping
        mapping = create_mapping(project_id=project_id)
        mappings[mapping['instance_uuid']] = mapping

        db_mappings = self.list_obj._get_by_project_id_from_db(
                self.context, project_id)
        for db_mapping in db_mappings:
            mapping = mappings[db_mapping.instance_uuid]
            for key in instance_mapping.InstanceMapping.fields.keys():
                self.assertEqual(db_mapping[key], mapping[key])

    def test_instance_mapping_list_get_by_cell_id(self):
        """Tests getting all of the InstanceMappings for a given CellMapping id
        """
        # we shouldn't have any instance mappings yet
        inst_mapping_list = (
            instance_mapping.InstanceMappingList.get_by_cell_id(
                self.context, sample_cell_mapping['id'])
        )
        self.assertEqual(0, len(inst_mapping_list))
        # now create an instance mapping in a cell
        db_inst_mapping1 = create_mapping()
        # let's also create an instance mapping that's not in a cell to make
        # sure our filtering is working
        db_inst_mapping2 = create_mapping(cell_id=None)
        self.assertIsNone(db_inst_mapping2['cell_id'])
        # now we should list out one instance mapping for the cell
        inst_mapping_list = (
            instance_mapping.InstanceMappingList.get_by_cell_id(
                self.context, db_inst_mapping1['cell_id'])
        )
        self.assertEqual(1, len(inst_mapping_list))
        self.assertEqual(db_inst_mapping1['id'], inst_mapping_list[0].id)

    def test_instance_mapping_get_by_instance_uuids(self):
        db_inst_mapping1 = create_mapping()
        db_inst_mapping2 = create_mapping(cell_id=None)
        # Create a third that we won't include
        create_mapping()
        uuids = [db_inst_mapping1.instance_uuid,
                 db_inst_mapping2.instance_uuid]
        mappings = instance_mapping.InstanceMappingList.get_by_instance_uuids(
            self.context, uuids + [uuidsentinel.deleted_instance])
        self.assertEqual(sorted(uuids),
                         sorted([m.instance_uuid for m in mappings]))

    def test_get_not_deleted_by_cell_and_project(self):
        cells = []
        # Create two cells
        for uuid in (uuidsentinel.cell1, uuidsentinel.cell2):
            cm = cell_mapping.CellMapping(context=self.context, uuid=uuid,
                                          database_connection="fake:///",
                                          transport_url='fake://')
            cm.create()
            cells.append(cm)

        uuids = {cells[0]: [uuidsentinel.c1i1, uuidsentinel.c1i2],
                 cells[1]: [uuidsentinel.c2i1, uuidsentinel.c2i2]}
        project_ids = ['fake-project-1', 'fake-project-2']
        # Create five instance_mappings such that:
        for cell, uuid in uuids.items():
            # Both the cells contain a mapping belonging to fake-project-1
            im1 = instance_mapping.InstanceMapping(context=self.context,
                project_id=project_ids[0], cell_mapping=cell,
                instance_uuid=uuid[0], queued_for_delete=False)
            im1.create()
            # Both the cells contain a mapping belonging to fake-project-2
            im2 = instance_mapping.InstanceMapping(context=self.context,
                project_id=project_ids[1], cell_mapping=cell,
                instance_uuid=uuid[1], queued_for_delete=False)
            im2.create()
            # The second cell has a third mapping that is queued for deletion
            # which belongs to fake-project-1.
            if cell.uuid == uuidsentinel.cell2:
                im3 = instance_mapping.InstanceMapping(context=self.context,
                    project_id=project_ids[0], cell_mapping=cell,
                    instance_uuid=uuidsentinel.qfd, queued_for_delete=True)
                im3.create()

        # Get not queued for deletion mappings from cell1 belonging to
        # fake-project-2.
        ims = (instance_mapping.InstanceMappingList.
               get_not_deleted_by_cell_and_project(
               self.context, cells[0].uuid, 'fake-project-2'))
        # This will give us one mapping from cell1
        self.assertEqual([uuidsentinel.c1i2],
                         sorted([m.instance_uuid for m in ims]))
        self.assertIn('cell_mapping', ims[0])
        # Get not queued for deletion mappings from cell2 belonging to
        # fake-project-1.
        ims = (instance_mapping.InstanceMappingList.
               get_not_deleted_by_cell_and_project(
               self.context, cells[1].uuid, 'fake-project-1'))
        # This will give us one mapping from cell2. Note that even if
        # there are two mappings belonging to fake-project-1 inside cell2,
        # only the one not queued for deletion is returned.
        self.assertEqual([uuidsentinel.c2i1],
                         sorted([m.instance_uuid for m in ims]))
        # Try getting a mapping belonging to a non-existing project_id.
        ims = (instance_mapping.InstanceMappingList.
               get_not_deleted_by_cell_and_project(
               self.context, cells[0].uuid, 'fake-project-3'))
        # Since no mappings belong to fake-project-3, nothing is returned.
        self.assertEqual([], sorted([m.instance_uuid for m in ims]))

    def test_get_not_deleted_by_cell_and_project_limit(self):
        cm = cell_mapping.CellMapping(context=self.context,
                                      uuid=uuidsentinel.cell,
                                      database_connection='fake:///',
                                      transport_url='fake://')
        cm.create()
        pid = self.context.project_id
        for uuid in (uuidsentinel.uuid2, uuidsentinel.inst2):
            im = instance_mapping.InstanceMapping(context=self.context,
                                                  project_id=pid,
                                                  cell_mapping=cm,
                                                  instance_uuid=uuid,
                                                  queued_for_delete=False)
            im.create()

        ims = (instance_mapping.InstanceMappingList.
               get_not_deleted_by_cell_and_project(self.context,
                                                   cm.uuid,
                                                   pid))
        self.assertEqual(2, len(ims))

        ims = (instance_mapping.InstanceMappingList.
               get_not_deleted_by_cell_and_project(self.context,
                                                   cm.uuid,
                                                   pid,
                                                   limit=10))
        self.assertEqual(2, len(ims))

        ims = (instance_mapping.InstanceMappingList.
               get_not_deleted_by_cell_and_project(self.context,
                                                   cm.uuid,
                                                   pid,
                                                   limit=1))
        self.assertEqual(1, len(ims))

    def test_get_not_deleted_by_cell_and_project_None(self):
        cm = cell_mapping.CellMapping(context=self.context,
                                      uuid=uuidsentinel.cell,
                                      database_connection='fake:///',
                                      transport_url='fake://')
        cm.create()
        im1 = instance_mapping.InstanceMapping(context=self.context,
                                               project_id='fake-project-1',
                                               cell_mapping=cm,
                                               instance_uuid=uuidsentinel.uid1,
                                               queued_for_delete=False)
        im1.create()
        im2 = instance_mapping.InstanceMapping(context=self.context,
                                               project_id='fake-project-2',
                                               cell_mapping=cm,
                                               instance_uuid=uuidsentinel.uid2,
                                               queued_for_delete=None)
        im2.create()
        # testing if it accepts None project_id in the query and
        # catches None queued for delete records.
        ims = (instance_mapping.InstanceMappingList.
               get_not_deleted_by_cell_and_project(self.context,
                                                   cm.uuid,
                                                   None))
        self.assertEqual(2, len(ims))

    def test_get_counts(self):
        create_mapping(project_id='fake-project', user_id='fake-user',
                       queued_for_delete=False)
        # mapping with another user
        create_mapping(project_id='fake-project', user_id='other-user',
                       queued_for_delete=False)
        # mapping in another project
        create_mapping(project_id='other-project', user_id='fake-user',
                       queued_for_delete=False)
        # queued_for_delete=True, should not be counted
        create_mapping(project_id='fake-project', user_id='fake-user',
                       queued_for_delete=True)
        # queued_for_delete=None (not yet migrated), should not be counted
        create_mapping(project_id='fake-project', user_id='fake-user',
                       queued_for_delete=None)

        # Count only across a project
        counts = instance_mapping.InstanceMappingList.get_counts(
            self.context, 'fake-project')
        self.assertEqual(2, counts['project']['instances'])
        self.assertNotIn('user', counts)

        # Count across a project and a user
        counts = instance_mapping.InstanceMappingList.get_counts(
            self.context, 'fake-project', user_id='fake-user')
        self.assertEqual(2, counts['project']['instances'])
        self.assertEqual(1, counts['user']['instances'])