Add extra metadata fields skip

Problem description
===================
Some OpenStack APIs do not always return exactly the
same metadata for all resources. As an example, we
have the server API, which might not return the
'OS-EXT-SRV-ATTR:host' attribute; it depends on the
virtual machine status.

Therefore, if the operator configures to retrieve
the value of the 'OS-EXT-SRV-ATTR:host' attribute in
the response, when the VM is in the 'RUNNING' state,
it will work properly; however, if it is in 'ERROR'
or other state, it will throw a 'key not found'
error when we are collecting the metadata.

Proposal
========
To allow operators to skip the extra_metadata_fields
gathering based on the collected sample attributes. We
propose to add a new 'extra_metadata_fields_skip' in
the dynamic pollster YAML definition where operators
can define some rules to skip gathering extra_metadata
for some samples based on their attributes.

Change-Id: I40176328e1863283890870098418c0944a75bad9
Depends-On: https://review.opendev.org/c/openstack/ceilometer/+/852021

2 files changed, 251 insertions, 0 deletions

diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py
index c42b1848..d7803d32 100644
--- a/ceilometer/polling/dynamic_pollster.py
+++ b/ceilometer/polling/dynamic_pollster.py
@@ -93,6 +93,15 @@ def validate_response_handler(val):
                 "are [%s]" % (value, ', '.join(list(VALID_HANDLERS))))
 
 
+def validate_extra_metadata_skip_samples(val):
+    if not isinstance(val, list) or next(
+            filter(lambda v: not isinstance(v, dict), val), None):
+        raise declarative.DynamicPollsterDefinitionException(
+            "Invalid extra_metadata_fields_skip configuration."
+            " It must be a list of maps. Provided value: %s,"
+            " value type: %s." % (val, type(val).__name__))
+
+
 class ResponseHandlerChain(object):
     """Tries to convert a string to a dict using the response handlers"""
 
@@ -205,6 +214,7 @@ class PollsterSampleExtractor(object):
         self.generate_new_metadata_fields(
             metadata=metadata, pollster_definitions=pollster_definitions)
 
+        pollster_sample['metadata'] = metadata
         extra_metadata = self.definitions.retrieve_extra_metadata(
             kwargs['manager'], pollster_sample, kwargs['conf'])
 
@@ -518,6 +528,8 @@ class PollsterDefinitions(object):
         PollsterDefinition(name='extra_metadata_fields_cache_seconds',
                            default=3600),
         PollsterDefinition(name='extra_metadata_fields'),
+        PollsterDefinition(name='extra_metadata_fields_skip', default=[{}],
+                           validator=validate_extra_metadata_skip_samples),
         PollsterDefinition(name='response_handlers', default=['json'],
                            validator=validate_response_handler),
         PollsterDefinition(name='base_metadata', default={})
@@ -574,6 +586,39 @@ class PollsterDefinitions(object):
                 "Required fields %s not specified."
                 % missing, self.configurations)
 
+    def should_skip_extra_metadata(self, skip, sample):
+        match_msg = "Sample [%s] %smatches with configured" \
+                    " extra_metadata_fields_skip [%s]."
+        if skip == sample:
+            LOG.debug(match_msg, sample, "", skip)
+            return True
+        if not isinstance(skip, dict) or not isinstance(sample, dict):
+            LOG.debug(match_msg, sample, "not ", skip)
+            return False
+
+        for key in skip:
+            if key not in sample:
+                LOG.debug(match_msg, sample, "not ", skip)
+                return False
+            if not self.should_skip_extra_metadata(skip[key], sample[key]):
+                LOG.debug(match_msg, sample, "not ", skip)
+                return False
+
+        LOG.debug(match_msg, sample, "", skip)
+        return True
+
+    def skip_sample(self, request_sample, skips):
+        for skip in skips:
+            if not skip:
+                continue
+            if self.should_skip_extra_metadata(skip, request_sample):
+                LOG.debug("Skipping extra_metadata_field gathering for "
+                          "sample [%s] as defined in the "
+                          "extra_metadata_fields_skip [%s]", request_sample,
+                          skip)
+                return True
+        return False
+
     def retrieve_extra_metadata(self, manager, request_sample, pollster_conf):
         extra_metadata_fields = self.configurations['extra_metadata_fields']
         if extra_metadata_fields:
@@ -583,6 +628,9 @@ class PollsterDefinitions(object):
                 extra_metadata_fields = [extra_metadata_fields]
             for ext_metadata in extra_metadata_fields:
                 ext_metadata.setdefault(
+                    'extra_metadata_fields_skip',
+                    self.configurations['extra_metadata_fields_skip'])
+                ext_metadata.setdefault(
                     'sample_type', self.configurations['sample_type'])
                 ext_metadata.setdefault('unit', self.configurations['unit'])
                 ext_metadata.setdefault(
@@ -603,6 +651,11 @@ class PollsterDefinitions(object):
                     ext_metadata, conf=pollster_conf, cache_ttl=cache_ttl,
                     extra_metadata_responses_cache=response_cache,
                 )
+
+                skips = ext_metadata['extra_metadata_fields_skip']
+                if self.skip_sample(request_sample, skips):
+                    continue
+
                 resources = [None]
                 if ext_metadata.get('endpoint_type'):
                     resources = manager.discover([
diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
index 653f2df2..f85af729 100644
--- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
@@ -734,6 +734,171 @@ class TestDynamicPollster(base.BaseTestCase):
                           'project_meta': 'META3'})
 
     @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_extra_metadata_fields_skip(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        extra_metadata_fields = [{
+            'name': "project_name",
+            'endpoint_type': "identity",
+            'url_path': "'/v3/projects/' + str(sample['project_id'])",
+            'value': "name",
+        }, {
+            'name': "project_alias",
+            'endpoint_type': "identity",
+            'extra_metadata_fields_skip': [{
+                'value': 7777
+            }],
+            'url_path': "'/v3/projects/' + "
+                        "str(sample['p_name'])",
+            'value': "name",
+        }]
+        definitions['value_attribute'] = 'project_id'
+        definitions['metadata_fields'] = ['to_skip', 'p_name']
+        definitions['extra_metadata_fields'] = extra_metadata_fields
+        definitions['extra_metadata_fields_skip'] = [{
+            'metadata': {
+                'to_skip': 'skip1'
+            }
+        }, {
+            'value': 8888
+        }]
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '''
+                        {"projects": [
+                            {"project_id": 9999, "p_name": "project1",
+                            "to_skip": "skip1"},
+                            {"project_id": 8888, "p_name": "project2",
+                            "to_skip": "skip2"},
+                            {"project_id": 7777, "p_name": "project3",
+                            "to_skip": "skip3"},
+                            {"project_id": 6666, "p_name": "project4",
+                            "to_skip": "skip4"}]
+                        }
+                    '''
+
+        return_value9999 = self.FakeResponse()
+        return_value9999.status_code = requests.codes.ok
+        return_value9999._text = '''
+                        {"project":
+                            {"project_id": 9999, "name": "project1"}
+                        }
+                    '''
+
+        return_value8888 = self.FakeResponse()
+        return_value8888.status_code = requests.codes.ok
+        return_value8888._text = '''
+                        {"project":
+                            {"project_id": 8888, "name": "project2"}
+                        }
+                    '''
+
+        return_value7777 = self.FakeResponse()
+        return_value7777.status_code = requests.codes.ok
+        return_value7777._text = '''
+                        {"project":
+                            {"project_id": 7777, "name": "project3"}
+                        }
+                    '''
+
+        return_value6666 = self.FakeResponse()
+        return_value6666.status_code = requests.codes.ok
+        return_value6666._text = '''
+                        {"project":
+                            {"project_id": 6666, "name": "project4"}
+                        }
+                    '''
+
+        return_valueP1 = self.FakeResponse()
+        return_valueP1.status_code = requests.codes.ok
+        return_valueP1._text = '''
+                        {"project":
+                            {"project_id": 7777, "name": "p1"}
+                        }
+                    '''
+
+        return_valueP2 = self.FakeResponse()
+        return_valueP2.status_code = requests.codes.ok
+        return_valueP2._text = '''
+                        {"project":
+                            {"project_id": 7777, "name": "p2"}
+                        }
+                    '''
+
+        return_valueP3 = self.FakeResponse()
+        return_valueP3.status_code = requests.codes.ok
+        return_valueP3._text = '''
+                        {"project":
+                            {"project_id": 7777, "name": "p3"}
+                        }
+                    '''
+
+        return_valueP4 = self.FakeResponse()
+        return_valueP4.status_code = requests.codes.ok
+        return_valueP4._text = '''
+                        {"project":
+                            {"project_id": 6666, "name": "p4"}
+                        }
+                    '''
+
+        def get(url, *args, **kwargs):
+            if '9999' in url:
+                return return_value9999
+            if '8888' in url:
+                return return_value8888
+            if '7777' in url:
+                return return_value7777
+            if '6666' in url:
+                return return_value6666
+            if 'project1' in url:
+                return return_valueP1
+            if 'project2' in url:
+                return return_valueP2
+            if 'project3' in url:
+                return return_valueP3
+            if 'project4' in url:
+                return return_valueP4
+
+            return return_value
+
+        client_mock.session.get = get
+        manager = mock.Mock
+        manager._keystone = client_mock
+
+        def discover(*args, **kwargs):
+            return ["https://endpoint.server.name/"]
+
+        manager.discover = discover
+        samples = pollster.get_samples(
+            manager=manager, cache=None,
+            resources=["https://endpoint.server.name/"])
+
+        samples = list(samples)
+        self.assertEqual(4, len(samples))
+
+        self.assertEqual(samples[0].volume, 9999)
+        self.assertEqual(samples[1].volume, 8888)
+        self.assertEqual(samples[2].volume, 7777)
+
+        self.assertEqual(samples[0].resource_metadata,
+                         {'p_name': 'project1', 'project_alias': 'p1',
+                          'to_skip': 'skip1'})
+        self.assertEqual(samples[1].resource_metadata,
+                         {'p_name': 'project2', 'project_alias': 'p2',
+                          'to_skip': 'skip2'})
+        self.assertEqual(samples[2].resource_metadata,
+                         {'p_name': 'project3', 'project_name': 'project3',
+                          'to_skip': 'skip3'})
+        self.assertEqual(samples[3].resource_metadata,
+                         {'p_name': 'project4',
+                          'project_alias': 'p4',
+                          'project_name': 'project4',
+                          'to_skip': 'skip4'})
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
     def test_execute_request_extra_metadata_fields_different_requests(
             self, client_mock):
         definitions = copy.deepcopy(
@@ -865,6 +1030,39 @@ class TestDynamicPollster(base.BaseTestCase):
                          "Accepted values are [json, xml, text]",
                          str(exception))
 
+    def test_configure_extra_metadata_field_skip_invalid_value(self):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['extra_metadata_fields_skip'] = 'teste'
+
+        exception = self.assertRaises(
+            declarative.DynamicPollsterDefinitionException,
+            dynamic_pollster.DynamicPollster,
+            pollster_definitions=definitions)
+        self.assertEqual("DynamicPollsterDefinitionException None: "
+                         "Invalid extra_metadata_fields_skip configuration."
+                         " It must be a list of maps. Provided value: teste,"
+                         " value type: str.",
+                         str(exception))
+
+    def test_configure_extra_metadata_field_skip_invalid_sub_value(self):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['extra_metadata_fields_skip'] = [{'test': '1'},
+                                                     {'test': '2'},
+                                                     'teste']
+
+        exception = self.assertRaises(
+            declarative.DynamicPollsterDefinitionException,
+            dynamic_pollster.DynamicPollster,
+            pollster_definitions=definitions)
+        self.assertEqual("DynamicPollsterDefinitionException None: "
+                         "Invalid extra_metadata_fields_skip configuration."
+                         " It must be a list of maps. Provided value: "
+                         "[{'test': '1'}, {'test': '2'}, 'teste'], "
+                         "value type: list.",
+                         str(exception))
+
     def test_configure_response_handler_definition_invalid_type(self):
         definitions = copy.deepcopy(
             self.pollster_definition_only_required_fields)