From bff9879e3489bec94241afc0cdfc8472211f7aff Mon Sep 17 00:00:00 2001 From: Pedro Henrique Date: Thu, 1 Sep 2022 10:03:29 -0300 Subject: Add extra metadata fields skip Problem description =================== Some OpenStack APIs do not always return exactly the same metadata for all resources. As an example, we have the server API, which might not return the 'OS-EXT-SRV-ATTR:host' attribute; it depends on the virtual machine status. Therefore, if the operator configures to retrieve the value of the 'OS-EXT-SRV-ATTR:host' attribute in the response, when the VM is in the 'RUNNING' state, it will work properly; however, if it is in 'ERROR' or other state, it will throw a 'key not found' error when we are collecting the metadata. Proposal ======== To allow operators to skip the extra_metadata_fields gathering based on the collected sample attributes. We propose to add a new 'extra_metadata_fields_skip' in the dynamic pollster YAML definition where operators can define some rules to skip gathering extra_metadata for some samples based on their attributes. Change-Id: I40176328e1863283890870098418c0944a75bad9 Depends-On: https://review.opendev.org/c/openstack/ceilometer/+/852021 --- ceilometer/polling/dynamic_pollster.py | 53 ++++++ .../tests/unit/polling/test_dynamic_pollster.py | 198 +++++++++++++++++++++ 2 files changed, 251 insertions(+) (limited to 'ceilometer') diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py index c42b1848..d7803d32 100644 --- a/ceilometer/polling/dynamic_pollster.py +++ b/ceilometer/polling/dynamic_pollster.py @@ -93,6 +93,15 @@ def validate_response_handler(val): "are [%s]" % (value, ', '.join(list(VALID_HANDLERS)))) +def validate_extra_metadata_skip_samples(val): + if not isinstance(val, list) or next( + filter(lambda v: not isinstance(v, dict), val), None): + raise declarative.DynamicPollsterDefinitionException( + "Invalid extra_metadata_fields_skip configuration." + " It must be a list of maps. Provided value: %s," + " value type: %s." % (val, type(val).__name__)) + + class ResponseHandlerChain(object): """Tries to convert a string to a dict using the response handlers""" @@ -205,6 +214,7 @@ class PollsterSampleExtractor(object): self.generate_new_metadata_fields( metadata=metadata, pollster_definitions=pollster_definitions) + pollster_sample['metadata'] = metadata extra_metadata = self.definitions.retrieve_extra_metadata( kwargs['manager'], pollster_sample, kwargs['conf']) @@ -518,6 +528,8 @@ class PollsterDefinitions(object): PollsterDefinition(name='extra_metadata_fields_cache_seconds', default=3600), PollsterDefinition(name='extra_metadata_fields'), + PollsterDefinition(name='extra_metadata_fields_skip', default=[{}], + validator=validate_extra_metadata_skip_samples), PollsterDefinition(name='response_handlers', default=['json'], validator=validate_response_handler), PollsterDefinition(name='base_metadata', default={}) @@ -574,6 +586,39 @@ class PollsterDefinitions(object): "Required fields %s not specified." % missing, self.configurations) + def should_skip_extra_metadata(self, skip, sample): + match_msg = "Sample [%s] %smatches with configured" \ + " extra_metadata_fields_skip [%s]." + if skip == sample: + LOG.debug(match_msg, sample, "", skip) + return True + if not isinstance(skip, dict) or not isinstance(sample, dict): + LOG.debug(match_msg, sample, "not ", skip) + return False + + for key in skip: + if key not in sample: + LOG.debug(match_msg, sample, "not ", skip) + return False + if not self.should_skip_extra_metadata(skip[key], sample[key]): + LOG.debug(match_msg, sample, "not ", skip) + return False + + LOG.debug(match_msg, sample, "", skip) + return True + + def skip_sample(self, request_sample, skips): + for skip in skips: + if not skip: + continue + if self.should_skip_extra_metadata(skip, request_sample): + LOG.debug("Skipping extra_metadata_field gathering for " + "sample [%s] as defined in the " + "extra_metadata_fields_skip [%s]", request_sample, + skip) + return True + return False + def retrieve_extra_metadata(self, manager, request_sample, pollster_conf): extra_metadata_fields = self.configurations['extra_metadata_fields'] if extra_metadata_fields: @@ -582,6 +627,9 @@ class PollsterDefinitions(object): if not isinstance(extra_metadata_fields, (list, tuple)): extra_metadata_fields = [extra_metadata_fields] for ext_metadata in extra_metadata_fields: + ext_metadata.setdefault( + 'extra_metadata_fields_skip', + self.configurations['extra_metadata_fields_skip']) ext_metadata.setdefault( 'sample_type', self.configurations['sample_type']) ext_metadata.setdefault('unit', self.configurations['unit']) @@ -603,6 +651,11 @@ class PollsterDefinitions(object): ext_metadata, conf=pollster_conf, cache_ttl=cache_ttl, extra_metadata_responses_cache=response_cache, ) + + skips = ext_metadata['extra_metadata_fields_skip'] + if self.skip_sample(request_sample, skips): + continue + resources = [None] if ext_metadata.get('endpoint_type'): resources = manager.discover([ diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index 653f2df2..f85af729 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -733,6 +733,171 @@ class TestDynamicPollster(base.BaseTestCase): 'meta': 'm3', 'project_meta': 'META3'}) + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_extra_metadata_fields_skip( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + extra_metadata_fields = [{ + 'name': "project_name", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + str(sample['project_id'])", + 'value': "name", + }, { + 'name': "project_alias", + 'endpoint_type': "identity", + 'extra_metadata_fields_skip': [{ + 'value': 7777 + }], + 'url_path': "'/v3/projects/' + " + "str(sample['p_name'])", + 'value': "name", + }] + definitions['value_attribute'] = 'project_id' + definitions['metadata_fields'] = ['to_skip', 'p_name'] + definitions['extra_metadata_fields'] = extra_metadata_fields + definitions['extra_metadata_fields_skip'] = [{ + 'metadata': { + 'to_skip': 'skip1' + } + }, { + 'value': 8888 + }] + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = ''' + {"projects": [ + {"project_id": 9999, "p_name": "project1", + "to_skip": "skip1"}, + {"project_id": 8888, "p_name": "project2", + "to_skip": "skip2"}, + {"project_id": 7777, "p_name": "project3", + "to_skip": "skip3"}, + {"project_id": 6666, "p_name": "project4", + "to_skip": "skip4"}] + } + ''' + + return_value9999 = self.FakeResponse() + return_value9999.status_code = requests.codes.ok + return_value9999._text = ''' + {"project": + {"project_id": 9999, "name": "project1"} + } + ''' + + return_value8888 = self.FakeResponse() + return_value8888.status_code = requests.codes.ok + return_value8888._text = ''' + {"project": + {"project_id": 8888, "name": "project2"} + } + ''' + + return_value7777 = self.FakeResponse() + return_value7777.status_code = requests.codes.ok + return_value7777._text = ''' + {"project": + {"project_id": 7777, "name": "project3"} + } + ''' + + return_value6666 = self.FakeResponse() + return_value6666.status_code = requests.codes.ok + return_value6666._text = ''' + {"project": + {"project_id": 6666, "name": "project4"} + } + ''' + + return_valueP1 = self.FakeResponse() + return_valueP1.status_code = requests.codes.ok + return_valueP1._text = ''' + {"project": + {"project_id": 7777, "name": "p1"} + } + ''' + + return_valueP2 = self.FakeResponse() + return_valueP2.status_code = requests.codes.ok + return_valueP2._text = ''' + {"project": + {"project_id": 7777, "name": "p2"} + } + ''' + + return_valueP3 = self.FakeResponse() + return_valueP3.status_code = requests.codes.ok + return_valueP3._text = ''' + {"project": + {"project_id": 7777, "name": "p3"} + } + ''' + + return_valueP4 = self.FakeResponse() + return_valueP4.status_code = requests.codes.ok + return_valueP4._text = ''' + {"project": + {"project_id": 6666, "name": "p4"} + } + ''' + + def get(url, *args, **kwargs): + if '9999' in url: + return return_value9999 + if '8888' in url: + return return_value8888 + if '7777' in url: + return return_value7777 + if '6666' in url: + return return_value6666 + if 'project1' in url: + return return_valueP1 + if 'project2' in url: + return return_valueP2 + if 'project3' in url: + return return_valueP3 + if 'project4' in url: + return return_valueP4 + + return return_value + + client_mock.session.get = get + manager = mock.Mock + manager._keystone = client_mock + + def discover(*args, **kwargs): + return ["https://endpoint.server.name/"] + + manager.discover = discover + samples = pollster.get_samples( + manager=manager, cache=None, + resources=["https://endpoint.server.name/"]) + + samples = list(samples) + self.assertEqual(4, len(samples)) + + self.assertEqual(samples[0].volume, 9999) + self.assertEqual(samples[1].volume, 8888) + self.assertEqual(samples[2].volume, 7777) + + self.assertEqual(samples[0].resource_metadata, + {'p_name': 'project1', 'project_alias': 'p1', + 'to_skip': 'skip1'}) + self.assertEqual(samples[1].resource_metadata, + {'p_name': 'project2', 'project_alias': 'p2', + 'to_skip': 'skip2'}) + self.assertEqual(samples[2].resource_metadata, + {'p_name': 'project3', 'project_name': 'project3', + 'to_skip': 'skip3'}) + self.assertEqual(samples[3].resource_metadata, + {'p_name': 'project4', + 'project_alias': 'p4', + 'project_name': 'project4', + 'to_skip': 'skip4'}) + @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_extra_metadata_fields_different_requests( self, client_mock): @@ -865,6 +1030,39 @@ class TestDynamicPollster(base.BaseTestCase): "Accepted values are [json, xml, text]", str(exception)) + def test_configure_extra_metadata_field_skip_invalid_value(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['extra_metadata_fields_skip'] = 'teste' + + exception = self.assertRaises( + declarative.DynamicPollsterDefinitionException, + dynamic_pollster.DynamicPollster, + pollster_definitions=definitions) + self.assertEqual("DynamicPollsterDefinitionException None: " + "Invalid extra_metadata_fields_skip configuration." + " It must be a list of maps. Provided value: teste," + " value type: str.", + str(exception)) + + def test_configure_extra_metadata_field_skip_invalid_sub_value(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['extra_metadata_fields_skip'] = [{'test': '1'}, + {'test': '2'}, + 'teste'] + + exception = self.assertRaises( + declarative.DynamicPollsterDefinitionException, + dynamic_pollster.DynamicPollster, + pollster_definitions=definitions) + self.assertEqual("DynamicPollsterDefinitionException None: " + "Invalid extra_metadata_fields_skip configuration." + " It must be a list of maps. Provided value: " + "[{'test': '1'}, {'test': '2'}, 'teste'], " + "value type: list.", + str(exception)) + def test_configure_response_handler_definition_invalid_type(self): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) -- cgit v1.2.1