path: root/nova/tests/unit/compute/test_multi_cell_list.py

#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from contextlib import contextmanager
import copy
import datetime
import mock
from oslo_utils.fixture import uuidsentinel as uuids

from nova.compute import multi_cell_list
from nova import context
from nova import exception
from nova import objects
from nova import test


class TestUtils(test.NoDBTestCase):
    def test_compare_simple(self):
        dt1 = datetime.datetime(2015, 11, 5, 20, 30, 00)
        dt2 = datetime.datetime(1955, 10, 25, 1, 21, 00)

        inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456, 'key4': dt1}
        inst2 = {'key0': 'foo', 'key1': 's', 'key2': 123, 'key4': dt2}

        # Equal key0, inst == inst2
        ctx = multi_cell_list.RecordSortContext(['key0'], ['asc'])
        self.assertEqual(0, ctx.compare_records(inst1, inst2))

        # Equal key0, inst == inst2 (direction should not matter)
        ctx = multi_cell_list.RecordSortContext(['key0'], ['desc'])
        self.assertEqual(0, ctx.compare_records(inst1, inst2))

        # Ascending by key1, inst1 < inst2
        ctx = multi_cell_list.RecordSortContext(['key1'], ['asc'])
        self.assertEqual(-1, ctx.compare_records(inst1, inst2))

        # Descending by key1, inst2 < inst1
        ctx = multi_cell_list.RecordSortContext(['key1'], ['desc'])
        self.assertEqual(1, ctx.compare_records(inst1, inst2))

        # Ascending by key2, inst2 < inst1
        ctx = multi_cell_list.RecordSortContext(['key2'], ['asc'])
        self.assertEqual(1, ctx.compare_records(inst1, inst2))

        # Descending by key2, inst1 < inst2
        ctx = multi_cell_list.RecordSortContext(['key2'], ['desc'])
        self.assertEqual(-1, ctx.compare_records(inst1, inst2))

        # Ascending by key4, inst1 > inst2
        ctx = multi_cell_list.RecordSortContext(['key4'], ['asc'])
        self.assertEqual(1, ctx.compare_records(inst1, inst2))

        # Descending by key4, inst1 < inst2
        ctx = multi_cell_list.RecordSortContext(['key4'], ['desc'])
        self.assertEqual(-1, ctx.compare_records(inst1, inst2))

    def test_compare_multiple(self):
        # key0 should not affect ordering, but key1 should

        inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456}
        inst2 = {'key0': 'foo', 'key1': 's', 'key2': 123}

        # Should be equivalent to ascending by key1
        ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
                                                ['asc', 'asc'])
        self.assertEqual(-1, ctx.compare_records(inst1, inst2))

        # Should be equivalent to descending by key1
        ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
                                                ['asc', 'desc'])
        self.assertEqual(1, ctx.compare_records(inst1, inst2))

    def test_wrapper(self):
        inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456}
        inst2 = {'key0': 'foo', 'key1': 's', 'key2': 123}

        ctx = context.RequestContext()
        ctx.cell_uuid = uuids.cell

        # Should sort by key1
        sort_ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
                                                     ['asc', 'asc'])
        iw1 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst1)
        iw2 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst2)
        # Check this both ways to make sure we're comparing against -1
        # and not just nonzero return from cmp()
        self.assertTrue(iw1 < iw2)
        self.assertFalse(iw2 < iw1)

        # Should sort reverse by key1
        sort_ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
                                                     ['asc', 'desc'])
        iw1 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst1)
        iw2 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst2)
        # Check this both ways to make sure we're comparing against -1
        # and not just nonzero return from cmp()
        self.assertTrue(iw1 > iw2)
        self.assertFalse(iw2 > iw1)

        # Make sure we can tell which cell a request came from
        self.assertEqual(uuids.cell, iw1.cell_uuid)

    def test_wrapper_sentinels(self):
        inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456}

        ctx = context.RequestContext()
        ctx.cell_uuid = uuids.cell

        sort_ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
                                                     ['asc', 'asc'])
        iw1 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst1)

        # Wrappers with sentinels
        iw2 = multi_cell_list.RecordWrapper(ctx, sort_ctx,
                                            context.did_not_respond_sentinel)
        iw3 = multi_cell_list.RecordWrapper(ctx, sort_ctx,
                                            exception.InstanceNotFound(
                                                instance_id='fake'))

        # NOTE(danms): The sentinel wrappers always win
        self.assertTrue(iw2 < iw1)
        self.assertTrue(iw3 < iw1)
        self.assertFalse(iw1 < iw2)
        self.assertFalse(iw1 < iw3)

        # NOTE(danms): Comparing two wrappers with sentinels will always return
        # True for less-than because we're just naive about always favoring the
        # left hand side. This is fine for our purposes but put it here to make
        # it explicit.
        self.assertTrue(iw2 < iw3)
        self.assertTrue(iw3 < iw2)

    def test_query_wrapper_success(self):
        def test(ctx, data):
            for thing in data:
                yield thing

        self.assertEqual([1, 2, 3],
                         list(multi_cell_list.query_wrapper(
                             None, test, [1, 2, 3])))

    def test_query_wrapper_timeout(self):
        def test(ctx):
            raise exception.CellTimeout

        self.assertEqual([context.did_not_respond_sentinel],
                         [x._db_record for x in
                          multi_cell_list.query_wrapper(
                              mock.MagicMock(), test)])

    def test_query_wrapper_fail(self):
        def tester(ctx):
            raise test.TestingException

        self.assertIsInstance(
            # query_wrapper is a generator so we convert to a list and
            # check the type on the first and only result
            [x._db_record for x in multi_cell_list.query_wrapper(
                mock.MagicMock(), tester)][0],
            test.TestingException)


class TestListContext(multi_cell_list.RecordSortContext):
    def compare_records(self, rec1, rec2):
        return -1


class TestLister(multi_cell_list.CrossCellLister):
    CONTEXT_CLS = TestListContext

    def __init__(self, data, sort_keys, sort_dirs,
                 cells=None, batch_size=None):
        self._data = data
        self._count_by_cell = {}
        super(TestLister, self).__init__(self.CONTEXT_CLS(sort_keys,
                                                          sort_dirs),
                                         cells=cells, batch_size=batch_size)

    @property
    def marker_identifier(self):
        return 'id'

    def _method_called(self, ctx, method, arg):
        self._count_by_cell.setdefault(ctx.cell_uuid, {})
        self._count_by_cell[ctx.cell_uuid].setdefault(method, [])
        self._count_by_cell[ctx.cell_uuid][method].append(arg)

    def call_summary(self, method):
        results = {
            'total': 0,
            'count_by_cell': [],
            'limit_by_cell': [],
            'total_by_cell': [],
            'called_in_cell': [],
        }
        for i, cell in enumerate(self._count_by_cell):
            if method not in self._count_by_cell[cell]:
                continue
            results['total'] += len(self._count_by_cell[cell][method])
            # List of number of calls in each cell
            results['count_by_cell'].append(
                len(self._count_by_cell[cell][method]))
            # List of limits used in calls to each cell
            results['limit_by_cell'].append(
                self._count_by_cell[cell][method])
            try:
                # List of total results fetched from each cell
                results['total_by_cell'].append(sum(
                    self._count_by_cell[cell][method]))
            except TypeError:
                # Don't do this for non-integer args
                pass
            results['called_in_cell'].append(cell)
        results['count_by_cell'].sort()
        results['limit_by_cell'].sort()
        results['total_by_cell'].sort()
        results['called_in_cell'].sort()
        return results

    def get_marker_record(self, ctx, marker):
        self._method_called(ctx, 'get_marker_record', marker)
        # Always assume this came from the second cell
        cell = self.cells[1]
        return cell.uuid, self._data[0]

    def get_marker_by_values(self, ctx, values):
        self._method_called(ctx, 'get_marker_by_values', values)
        return self._data[0]

    def get_by_filters(self, ctx, filters, limit, marker, **kwargs):
        self._method_called(ctx, 'get_by_filters', limit)
        if 'batch_size' in kwargs:
            count = min(kwargs['batch_size'], limit)
        else:
            count = limit
        batch = self._data[:count]
        self._data = self._data[count:]
        return batch


@contextmanager
def target_cell_cheater(context, target_cell):
    # In order to help us do accounting, we need to mimic the real
    # behavior where at least cell_uuid gets set on the context, which
    # doesn't happen in the simple test fixture.
    context = copy.deepcopy(context)
    context.cell_uuid = target_cell.uuid
    yield context


@mock.patch('nova.context.target_cell', new=target_cell_cheater)
class TestBatching(test.NoDBTestCase):
    def setUp(self):
        super(TestBatching, self).setUp()

        self._data = [{'id': 'foo-%i' % i}
                      for i in range(0, 1000)]
        self._cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
                                           name='cell%i' % i)
                       for i in range(0, 10)]

    def test_batches_not_needed(self):
        lister = TestLister(self._data, [], [],
                            cells=self._cells, batch_size=10)
        ctx = context.RequestContext()
        res = list(lister.get_records_sorted(ctx, {}, 5, None))
        self.assertEqual(5, len(res))
        summary = lister.call_summary('get_by_filters')
        # We only needed one batch per cell to hit the total,
        # so we should have the same number of calls as cells
        self.assertEqual(len(self._cells), summary['total'])
        # One call per cell, hitting all cells
        self.assertEqual(len(self._cells), len(summary['count_by_cell']))
        self.assertTrue(all([
            cell_count == 1 for cell_count in summary['count_by_cell']]))

    def test_batches(self):
        lister = TestLister(self._data, [], [],
                            cells=self._cells, batch_size=10)
        ctx = context.RequestContext()
        res = list(lister.get_records_sorted(ctx, {}, 500, None))
        self.assertEqual(500, len(res))
        summary = lister.call_summary('get_by_filters')

        # Since we got everything from one cell (due to how things are sorting)
        # we should have made 500 / 10 calls to one cell, and 1 call to
        # the rest
        calls_expected = [1 for cell in self._cells[1:]] + [500 / 10]
        self.assertEqual(calls_expected, summary['count_by_cell'])

        # Since we got everything from one cell (due to how things are sorting)
        # we should have received 500 from one cell and 10 from the rest
        count_expected = [10 for cell in self._cells[1:]] + [500]
        self.assertEqual(count_expected, summary['total_by_cell'])

        # Since we got everything from one cell (due to how things are sorting)
        # we should have a bunch of calls for batches of 10, one each for
        # every cell except the one that served the bulk of the requests which
        # should have 500 / 10 batches of 10.
        limit_expected = ([[10] for cell in self._cells[1:]] +
                          [[10 for i in range(0, 500 // 10)]])
        self.assertEqual(limit_expected, summary['limit_by_cell'])

    def test_no_batches(self):
        lister = TestLister(self._data, [], [],
                            cells=self._cells)
        ctx = context.RequestContext()
        res = list(lister.get_records_sorted(ctx, {}, 50, None))
        self.assertEqual(50, len(res))
        summary = lister.call_summary('get_by_filters')

        # Since we used no batches we should have one call per cell
        calls_expected = [1 for cell in self._cells]
        self.assertEqual(calls_expected, summary['count_by_cell'])

        # Since we used no batches, each cell should have returned 50 results
        count_expected = [50 for cell in self._cells]
        self.assertEqual(count_expected, summary['total_by_cell'])

        # Since we used no batches, each cell call should be for $limit
        limit_expected = [[count] for count in count_expected]
        self.assertEqual(limit_expected, summary['limit_by_cell'])


class FailureListContext(multi_cell_list.RecordSortContext):
    def compare_records(self, rec1, rec2):
        return 0


class FailureLister(TestLister):
    CONTEXT_CLS = FailureListContext

    def __init__(self, *a, **k):
        super(FailureLister, self).__init__(*a, **k)
        self._fails = {}

    def set_fails(self, cell, fails):
        self._fails[cell] = fails

    def get_by_filters(self, ctx, *a, **k):
        try:
            action = self._fails[ctx.cell_uuid].pop(0)
        except (IndexError, KeyError):
            action = None

        if action == context.did_not_respond_sentinel:
            raise exception.CellTimeout
        elif isinstance(action, Exception):
            raise test.TestingException
        else:
            return super(FailureLister, self).get_by_filters(ctx, *a, **k)


@mock.patch('nova.context.target_cell', new=target_cell_cheater)
class TestBaseClass(test.NoDBTestCase):
    def test_with_failing_cells(self):
        data = [{'id': 'foo-%i' % i} for i in range(0, 100)]
        cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
                                     name='cell%i' % i)
                 for i in range(0, 3)]

        lister = FailureLister(data, [], [], cells=cells)
        # Two of the cells will fail, one with timeout and one
        # with an error
        lister.set_fails(uuids.cell0, [context.did_not_respond_sentinel])
        # Note that InstanceNotFound exception will never appear during
        # instance listing, the aim is to only simulate a situation where
        # there could be some type of exception arising.
        lister.set_fails(uuids.cell1, exception.InstanceNotFound(
            instance_id='fake'))
        ctx = context.RequestContext()
        result = lister.get_records_sorted(ctx, {}, 50, None, batch_size=10)
        # We should still have 50 results since there are enough from the
        # good cells to fill our limit.
        self.assertEqual(50, len(list(result)))
        # Make sure the counts line up
        self.assertEqual(1, len(lister.cells_failed))
        self.assertEqual(1, len(lister.cells_timed_out))
        self.assertEqual(1, len(lister.cells_responded))

    def test_with_failing_middle_cells(self):
        data = [{'id': 'foo-%i' % i} for i in range(0, 100)]
        cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
                                     name='cell%i' % i)
                 for i in range(0, 3)]

        lister = FailureLister(data, [], [], cells=cells)
        # One cell will succeed and then time out, one will fail immediately,
        # and the last will always work
        lister.set_fails(uuids.cell0, [None, context.did_not_respond_sentinel])
        # Note that BuildAbortException will never appear during instance
        # listing, the aim is to only simulate a situation where there could
        # be some type of exception arising.
        lister.set_fails(uuids.cell1, exception.BuildAbortException(
            instance_uuid='fake', reason='fake'))
        ctx = context.RequestContext()
        result = lister.get_records_sorted(ctx, {}, 50, None,
                                           batch_size=5)
        # We should still have 50 results since there are enough from the
        # good cells to fill our limit.
        self.assertEqual(50, len(list(result)))
        # Make sure the counts line up
        self.assertEqual(1, len(lister.cells_responded))
        self.assertEqual(1, len(lister.cells_failed))
        self.assertEqual(1, len(lister.cells_timed_out))

    def test_marker_cell_not_requeried(self):
        data = [{'id': 'foo-%i' % i} for i in range(0, 100)]
        cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
                                     name='cell%i' % i)
                 for i in range(0, 3)]

        lister = TestLister(data, [], [], cells=cells)
        ctx = context.RequestContext()
        result = list(lister.get_records_sorted(ctx, {}, 10, None))
        result = list(lister.get_records_sorted(ctx, {}, 10, result[-1]['id']))

        # get_marker_record() is called untargeted and its result defines which
        # cell we skip.
        gmr_summary = lister.call_summary('get_marker_record')
        self.assertEqual([None], gmr_summary['called_in_cell'])

        # All cells other than the second one should have been called for
        # a local marker
        gmbv_summary = lister.call_summary('get_marker_by_values')
        self.assertEqual(sorted([cell.uuid for cell in cells
                                 if cell.uuid != uuids.cell1]),
                         gmbv_summary['called_in_cell'])