diff options
Diffstat (limited to 'ceilometer')
-rw-r--r-- | ceilometer/polling/dynamic_pollster.py | 303 | ||||
-rw-r--r-- | ceilometer/tests/unit/polling/test_dynamic_pollster.py | 479 |
2 files changed, 654 insertions, 128 deletions
diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py index 8fd54a48..c9347ae4 100644 --- a/ceilometer/polling/dynamic_pollster.py +++ b/ceilometer/polling/dynamic_pollster.py @@ -20,6 +20,7 @@ import copy import json import re +import subprocess import time import xmltodict @@ -205,7 +206,7 @@ class PollsterSampleExtractor(object): metadata=metadata, pollster_definitions=pollster_definitions) extra_metadata = self.definitions.retrieve_extra_metadata( - kwargs['manager'], pollster_sample) + kwargs['manager'], pollster_sample, kwargs['conf']) for key in extra_metadata.keys(): if key in metadata.keys(): @@ -518,7 +519,8 @@ class PollsterDefinitions(object): default=3600), PollsterDefinition(name='extra_metadata_fields'), PollsterDefinition(name='response_handlers', default=['json'], - validator=validate_response_handler) + validator=validate_response_handler), + PollsterDefinition(name='base_metadata', default={}) ] extra_definitions = [] @@ -572,114 +574,75 @@ class PollsterDefinitions(object): "Required fields %s not specified." % missing, self.configurations) - def retrieve_extra_metadata(self, manager, request_sample): + def retrieve_extra_metadata(self, manager, request_sample, pollster_conf): extra_metadata_fields = self.configurations['extra_metadata_fields'] if extra_metadata_fields: - if isinstance(self, NonOpenStackApisPollsterDefinition): - raise declarative.NonOpenStackApisDynamicPollsterException( - "Not supported the use of extra metadata gathering for " - "non-openstack pollsters [%s] (yet)." - % self.configurations['name']) - - return self._retrieve_extra_metadata( - extra_metadata_fields, manager, request_sample) + extra_metadata_samples = {} + extra_metadata_by_name = {} + if not isinstance(extra_metadata_fields, (list, tuple)): + extra_metadata_fields = [extra_metadata_fields] + for ext_metadata in extra_metadata_fields: + ext_metadata.setdefault( + 'sample_type', self.configurations['sample_type']) + ext_metadata.setdefault('unit', self.configurations['unit']) + ext_metadata.setdefault( + 'value_attribute', ext_metadata.get( + 'value', self.configurations['value_attribute'])) + ext_metadata['base_metadata'] = { + 'extra_metadata_captured': extra_metadata_samples, + 'extra_metadata_by_name': extra_metadata_by_name, + 'sample': request_sample + } + parent_cache_ttl = self.configurations[ + 'extra_metadata_fields_cache_seconds'] + cache_ttl = ext_metadata.get( + 'extra_metadata_fields_cache_seconds', parent_cache_ttl + ) + response_cache = self.response_cache + extra_metadata_pollster = DynamicPollster( + ext_metadata, conf=pollster_conf, cache_ttl=cache_ttl, + extra_metadata_responses_cache=response_cache, + ) + resources = [None] + if ext_metadata.get('endpoint_type'): + resources = manager.discover([ + extra_metadata_pollster.default_discovery], {}) + samples = extra_metadata_pollster.get_samples( + manager, None, resources) + for sample in samples: + self.fill_extra_metadata_samples( + extra_metadata_by_name, + extra_metadata_samples, + sample) + return extra_metadata_samples LOG.debug("No extra metadata to be captured for pollsters [%s] and " "request sample [%s].", self.definitions, request_sample) return {} - def _retrieve_extra_metadata( - self, extra_metadata_fields, manager, request_sample): - LOG.debug("Processing extra metadata fields [%s] for " - "sample [%s].", extra_metadata_fields, - request_sample) - - extra_metadata_captured = {} - for extra_metadata in extra_metadata_fields: - extra_metadata_name = extra_metadata['name'] - - if extra_metadata_name in extra_metadata_captured.keys(): - LOG.warning("Duplicated extra metadata name [%s]. Therefore, " - "we do not process this iteration [%s].", - extra_metadata_name, extra_metadata) + def fill_extra_metadata_samples(self, extra_metadata_by_name, + extra_metadata_samples, sample): + extra_metadata_samples[sample.name] = sample.volume + LOG.debug("Merging the sample metadata [%s] of the " + "extra_metadata_field [%s], with the " + "extra_metadata_samples [%s].", + sample.resource_metadata, + sample.name, + extra_metadata_samples) + for key, value in sample.resource_metadata.items(): + if value is None and key in extra_metadata_samples: + LOG.debug("Metadata [%s] for extra_metadata_field [%s] " + "is None, skipping metadata override by None " + "value", key, sample.name) continue + extra_metadata_samples[key] = value + extra_metadata_by_name[sample.name] = { + 'value': sample.volume, + 'metadata': sample.resource_metadata + } - LOG.debug("Processing extra metadata [%s] for sample [%s].", - extra_metadata_name, request_sample) - - endpoint_type = 'endpoint:' + extra_metadata['endpoint_type'] - if not endpoint_type.endswith( - PollsterDefinitions.EXTERNAL_ENDPOINT_TYPE): - response = self.execute_openstack_extra_metadata_gathering( - endpoint_type, extra_metadata, manager, request_sample, - extra_metadata_captured) - else: - raise declarative.NonOpenStackApisDynamicPollsterException( - "Not supported the use of extra metadata gathering for " - "non-openstack endpoints [%s] (yet)." % extra_metadata) - - extra_metadata_extractor_kwargs = { - 'value_attribute': extra_metadata['value'], - 'sample': request_sample} - - extra_metadata_value = \ - self.sample_extractor.retrieve_attribute_nested_value( - response, **extra_metadata_extractor_kwargs) - - LOG.debug("Generated extra metadata [%s] with value [%s].", - extra_metadata_name, extra_metadata_value) - extra_metadata_captured[extra_metadata_name] = extra_metadata_value - - return extra_metadata_captured - - def execute_openstack_extra_metadata_gathering(self, endpoint_type, - extra_metadata, manager, - request_sample, - extra_metadata_captured): - url_for_endpoint_type = manager.discover( - [endpoint_type], self.response_cache) - - LOG.debug("URL [%s] found for endpoint type [%s].", - url_for_endpoint_type, endpoint_type) - - if url_for_endpoint_type: - url_for_endpoint_type = url_for_endpoint_type[0] - - self.sample_gatherer.generate_url_path( - extra_metadata, request_sample, extra_metadata_captured) - - cached_response, max_ttl_for_cache = self.response_cache.get( - extra_metadata['url_path'], (None, None)) - - extra_metadata_fields_cache_seconds = extra_metadata.get( - 'extra_metadata_fields_cache_seconds', - self.configurations['extra_metadata_fields_cache_seconds']) - - current_time = time.time() - if cached_response and max_ttl_for_cache >= current_time: - LOG.debug("Returning response [%s] for request [%s] as the TTL " - "[max=%s, current_time=%s] has not expired yet.", - cached_response, extra_metadata['url_path'], - max_ttl_for_cache, current_time) - return cached_response - - if cached_response: - LOG.debug("Cleaning cached response [%s] for request [%s] " - "as the TTL [max=%s, current_time=%s] has expired.", - cached_response, extra_metadata['url_path'], - max_ttl_for_cache, current_time) - - response = self.sample_gatherer.execute_request_for_definitions( - extra_metadata, **{'manager': manager, - 'keystone_client': manager._keystone, - 'resource': url_for_endpoint_type, - 'execute_id_overrides': False}) - - max_ttl_for_cache = time.time() + extra_metadata_fields_cache_seconds - - cache_tuple = (response, max_ttl_for_cache) - self.response_cache[extra_metadata['url_path']] = cache_tuple - return response + LOG.debug("extra_metadata_samples after merging: [%s].", + extra_metadata_samples) class MultiMetricPollsterDefinitions(PollsterDefinitions): @@ -739,6 +702,42 @@ class PollsterSampleGatherer(object): url_path=definitions.configurations['url_path'] ) + def get_cache_key(self, definitions, **kwargs): + return self.get_request_linked_samples_url(kwargs, definitions) + + def get_cached_response(self, definitions, **kwargs): + if self.definitions.cache_ttl == 0: + return + cache_key = self.get_cache_key(definitions, **kwargs) + response_cache = self.definitions.response_cache + cached_response, max_ttl_for_cache = response_cache.get( + cache_key, (None, None)) + + current_time = time.time() + if cached_response and max_ttl_for_cache >= current_time: + LOG.debug("Returning response [%s] for request [%s] as the TTL " + "[max=%s, current_time=%s] has not expired yet.", + cached_response, definitions, + max_ttl_for_cache, current_time) + return cached_response + + if cached_response and max_ttl_for_cache < current_time: + LOG.debug("Cleaning cached response [%s] for request [%s] " + "as the TTL [max=%s, current_time=%s] has expired.", + cached_response, definitions, + max_ttl_for_cache, current_time) + response_cache.pop(cache_key, None) + + def store_cached_response(self, definitions, resp, **kwargs): + if self.definitions.cache_ttl == 0: + return + cache_key = self.get_cache_key(definitions, **kwargs) + extra_metadata_fields_cache_seconds = self.definitions.cache_ttl + max_ttl_for_cache = time.time() + extra_metadata_fields_cache_seconds + + cache_tuple = (resp, max_ttl_for_cache) + self.definitions.response_cache[cache_key] = cache_tuple + @property def default_discovery(self): return 'endpoint:' + self.definitions.configurations['endpoint_type'] @@ -748,10 +747,14 @@ class PollsterSampleGatherer(object): self.definitions.configurations, **kwargs) def execute_request_for_definitions(self, definitions, **kwargs): - resp, url = self._internal_execute_request_get_samples( - definitions=definitions, **kwargs) + if response_dict := self.get_cached_response(definitions, **kwargs): + url = 'cached' + else: + resp, url = self._internal_execute_request_get_samples( + definitions=definitions, **kwargs) + response_dict = self.response_handler_chain.handle(resp.text) + self.store_cached_response(definitions, response_dict, **kwargs) - response_dict = self.response_handler_chain.handle(resp.text) entry_size = len(response_dict) LOG.debug("Entries [%s] in the DICT for request [%s] " "for dynamic pollster [%s].", @@ -790,21 +793,6 @@ class PollsterSampleGatherer(object): self.generate_new_attributes_in_sample( request_sample, resource_id_attribute, 'id') - def generate_url_path(self, extra_metadata, sample, - extra_metadata_captured): - if not extra_metadata.get('url_path_original'): - extra_metadata[ - 'url_path_original'] = extra_metadata['url_path'] - - extra_metadata['url_path'] = eval( - extra_metadata['url_path_original']) - - LOG.debug("URL [%s] generated for pattern [%s] for sample [%s] and " - "extra metadata captured [%s].", - extra_metadata['url_path'], - extra_metadata['url_path_original'], sample, - extra_metadata_captured) - def generate_new_attributes_in_sample( self, sample, attribute_key, new_attribute_key): @@ -881,6 +869,15 @@ class PollsterSampleGatherer(object): def get_request_url(self, kwargs, url_path): endpoint = kwargs['resource'] + params = copy.deepcopy( + self.definitions.configurations.get( + 'base_metadata', {})) + try: + url_path = eval(url_path, params) + except Exception: + LOG.debug("Cannot eval path [%s] with params [%s]," + " using [%s] instead.", + url_path, params, url_path) return urlparse.urljoin(endpoint, url_path) def retrieve_entries_from_response(self, response_json, definitions): @@ -919,6 +916,57 @@ class NonOpenStackApisPollsterDefinition(PollsterDefinitions): return configurations.get('module') +class HostCommandPollsterDefinition(PollsterDefinitions): + + extra_definitions = [ + PollsterDefinition(name='endpoint_type', required=False), + PollsterDefinition(name='url_path', required=False), + PollsterDefinition(name='host_command', required=True)] + + def __init__(self, configurations): + super(HostCommandPollsterDefinition, self).__init__( + configurations) + self.sample_gatherer = HostCommandSamplesGatherer(self) + + @staticmethod + def is_field_applicable_to_definition(configurations): + return configurations.get('host_command') + + +class HostCommandSamplesGatherer(PollsterSampleGatherer): + + class Response(object): + def __init__(self, text): + self.text = text + + def get_cache_key(self, definitions, **kwargs): + return self.get_command(definitions) + + def _internal_execute_request_get_samples(self, definitions, **kwargs): + command = self.get_command(definitions, **kwargs) + LOG.debug('Running Host command: [%s]', command) + result = subprocess.getoutput(command) + LOG.debug('Host command [%s] result: [%s]', command, result) + return self.Response(result), command + + def get_command(self, definitions, next_sample_url=None, **kwargs): + command = next_sample_url or definitions['host_command'] + params = copy.deepcopy( + self.definitions.configurations.get( + 'base_metadata', {})) + try: + command = eval(command, params) + except Exception: + LOG.debug("Cannot eval command [%s] with params [%s]," + " using [%s] instead.", + command, params, command) + return command + + @property + def default_discovery(self): + return 'local_node' + + class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer): @property @@ -1010,8 +1058,10 @@ class DynamicPollster(plugin_base.PollsterBase): # Mandatory name field name = "" - def __init__(self, pollster_definitions={}, conf=None, - supported_definitions=[NonOpenStackApisPollsterDefinition, + def __init__(self, pollster_definitions={}, conf=None, cache_ttl=0, + extra_metadata_responses_cache=None, + supported_definitions=[HostCommandPollsterDefinition, + NonOpenStackApisPollsterDefinition, MultiMetricPollsterDefinitions, SingleMetricPollsterDefinitions]): super(DynamicPollster, self).__init__(conf) @@ -1021,6 +1071,10 @@ class DynamicPollster(plugin_base.PollsterBase): self.definitions = PollsterDefinitionBuilder( self.supported_definitions).build_definitions(pollster_definitions) + self.definitions.cache_ttl = cache_ttl + self.definitions.response_cache = extra_metadata_responses_cache + if extra_metadata_responses_cache is None: + self.definitions.response_cache = {} self.pollster_definitions = self.definitions.configurations if 'metadata_fields' in self.pollster_definitions: LOG.debug("Metadata fields configured to [%s].", @@ -1054,9 +1108,12 @@ class DynamicPollster(plugin_base.PollsterBase): for r in resources: LOG.debug("Executing get sample for resource [%s].", r) samples = self.load_samples(r, manager) + if not isinstance(samples, (list, tuple)): + samples = [samples] for pollster_sample in samples: - kwargs = {'manager': manager, 'resource': r} - sample = self.extract_sample(pollster_sample, **kwargs) + sample = self.extract_sample( + pollster_sample, manager=manager, + resource=r, conf=self.conf) if isinstance(sample, SkippedSample): continue yield from sample diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index dfc0c576..653f2df2 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -389,6 +389,424 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(4, len(samples)) @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_extra_metadata_fields_cache_disabled( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + extra_metadata_fields = { + 'extra_metadata_fields_cache_seconds': 0, + 'name': "project_name", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + str(sample['project_id'])", + 'value': "name", + } + definitions['value_attribute'] = 'project_id' + definitions['extra_metadata_fields'] = extra_metadata_fields + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = ''' + {"projects": [ + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}, + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}, + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}] + } + ''' + + return_value9999 = self.FakeResponse() + return_value9999.status_code = requests.codes.ok + return_value9999._text = ''' + {"project": + {"project_id": 9999, "name": "project1"} + } + ''' + + return_value8888 = self.FakeResponse() + return_value8888.status_code = requests.codes.ok + return_value8888._text = ''' + {"project": + {"project_id": 8888, "name": "project2"} + } + ''' + + return_value7777 = self.FakeResponse() + return_value7777.status_code = requests.codes.ok + return_value7777._text = ''' + {"project": + {"project_id": 7777, "name": "project3"} + } + ''' + + def get(url, *args, **kwargs): + if '9999' in url: + return return_value9999 + if '8888' in url: + return return_value8888 + if '7777' in url: + return return_value7777 + return return_value + + client_mock.session.get.side_effect = get + manager = mock.Mock + manager._keystone = client_mock + + def discover(*args, **kwargs): + return ["https://endpoint.server.name/"] + + manager.discover = discover + samples = pollster.get_samples( + manager=manager, cache=None, + resources=["https://endpoint.server.name/"]) + + samples = list(samples) + + n_calls = client_mock.session.get.call_count + self.assertEqual(9, len(samples)) + self.assertEqual(10, n_calls) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_extra_metadata_fields_cache_enabled( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + extra_metadata_fields = { + 'extra_metadata_fields_cache_seconds': 3600, + 'name': "project_name", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + str(sample['project_id'])", + 'value': "name", + } + definitions['value_attribute'] = 'project_id' + definitions['extra_metadata_fields'] = extra_metadata_fields + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = ''' + {"projects": [ + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}, + {"project_id": 9999, "name": "project4"}, + {"project_id": 8888, "name": "project5"}, + {"project_id": 7777, "name": "project6"}, + {"project_id": 9999, "name": "project7"}, + {"project_id": 8888, "name": "project8"}, + {"project_id": 7777, "name": "project9"}] + } + ''' + + return_value9999 = self.FakeResponse() + return_value9999.status_code = requests.codes.ok + return_value9999._text = ''' + {"project": + {"project_id": 9999, "name": "project1"} + } + ''' + + return_value8888 = self.FakeResponse() + return_value8888.status_code = requests.codes.ok + return_value8888._text = ''' + {"project": + {"project_id": 8888, "name": "project2"} + } + ''' + + return_value7777 = self.FakeResponse() + return_value7777.status_code = requests.codes.ok + return_value7777._text = ''' + {"project": + {"project_id": 7777, "name": "project3"} + } + ''' + + def get(url, *args, **kwargs): + if '9999' in url: + return return_value9999 + if '8888' in url: + return return_value8888 + if '7777' in url: + return return_value7777 + return return_value + + client_mock.session.get.side_effect = get + manager = mock.Mock + manager._keystone = client_mock + + def discover(*args, **kwargs): + return ["https://endpoint.server.name/"] + + manager.discover = discover + samples = pollster.get_samples( + manager=manager, cache=None, + resources=["https://endpoint.server.name/"]) + + samples = list(samples) + + n_calls = client_mock.session.get.call_count + self.assertEqual(9, len(samples)) + self.assertEqual(4, n_calls) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_extra_metadata_fields( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + extra_metadata_fields = [{ + 'name': "project_name", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + str(sample['project_id'])", + 'value': "name", + 'metadata_fields': ['meta'] + }, { + 'name': "project_alias", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + " + "str(extra_metadata_captured['project_name'])", + 'value': "name", + 'metadata_fields': ['meta'] + }, { + 'name': "project_meta", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + " + "str(extra_metadata_by_name['project_name']" + "['metadata']['meta'])", + 'value': "project_id", + 'metadata_fields': ['meta'] + }] + definitions['value_attribute'] = 'project_id' + definitions['extra_metadata_fields'] = extra_metadata_fields + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = ''' + {"projects": [ + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}] + } + ''' + + return_value9999 = self.FakeResponse() + return_value9999.status_code = requests.codes.ok + return_value9999._text = ''' + {"project": + {"project_id": 9999, "name": "project1", + "meta": "m1"} + } + ''' + + return_value8888 = self.FakeResponse() + return_value8888.status_code = requests.codes.ok + return_value8888._text = ''' + {"project": + {"project_id": 8888, "name": "project2", + "meta": "m2"} + } + ''' + + return_value7777 = self.FakeResponse() + return_value7777.status_code = requests.codes.ok + return_value7777._text = ''' + {"project": + {"project_id": 7777, "name": "project3", + "meta": "m3"} + } + ''' + + return_valueP1 = self.FakeResponse() + return_valueP1.status_code = requests.codes.ok + return_valueP1._text = ''' + {"project": + {"project_id": 7777, "name": "p1", + "meta": null} + } + ''' + + return_valueP2 = self.FakeResponse() + return_valueP2.status_code = requests.codes.ok + return_valueP2._text = ''' + {"project": + {"project_id": 7777, "name": "p2", + "meta": null} + } + ''' + + return_valueP3 = self.FakeResponse() + return_valueP3.status_code = requests.codes.ok + return_valueP3._text = ''' + {"project": + {"project_id": 7777, "name": "p3", + "meta": null} + } + ''' + + return_valueM1 = self.FakeResponse() + return_valueM1.status_code = requests.codes.ok + return_valueM1._text = ''' + {"project": + {"project_id": "META1", "name": "p3", + "meta": null} + } + ''' + + return_valueM2 = self.FakeResponse() + return_valueM2.status_code = requests.codes.ok + return_valueM2._text = ''' + {"project": + {"project_id": "META2", "name": "p3", + "meta": null} + } + ''' + + return_valueM3 = self.FakeResponse() + return_valueM3.status_code = requests.codes.ok + return_valueM3._text = ''' + {"project": + {"project_id": "META3", "name": "p3", + "meta": null} + } + ''' + + def get(url, *args, **kwargs): + if '9999' in url: + return return_value9999 + if '8888' in url: + return return_value8888 + if '7777' in url: + return return_value7777 + if 'project1' in url: + return return_valueP1 + if 'project2' in url: + return return_valueP2 + if 'project3' in url: + return return_valueP3 + if 'm1' in url: + return return_valueM1 + if 'm2' in url: + return return_valueM2 + if 'm3' in url: + return return_valueM3 + + return return_value + + client_mock.session.get = get + manager = mock.Mock + manager._keystone = client_mock + + def discover(*args, **kwargs): + return ["https://endpoint.server.name/"] + + manager.discover = discover + samples = pollster.get_samples( + manager=manager, cache=None, + resources=["https://endpoint.server.name/"]) + + samples = list(samples) + self.assertEqual(3, len(samples)) + + self.assertEqual(samples[0].volume, 9999) + self.assertEqual(samples[1].volume, 8888) + self.assertEqual(samples[2].volume, 7777) + + self.assertEqual(samples[0].resource_metadata, + {'project_name': 'project1', + 'project_alias': 'p1', + 'meta': 'm1', + 'project_meta': 'META1'}) + self.assertEqual(samples[1].resource_metadata, + {'project_name': 'project2', + 'project_alias': 'p2', + 'meta': 'm2', + 'project_meta': 'META2'}) + self.assertEqual(samples[2].resource_metadata, + {'project_name': 'project3', + 'project_alias': 'p3', + 'meta': 'm3', + 'project_meta': 'META3'}) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_extra_metadata_fields_different_requests( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + + command = ''' \'\'\'echo '{"project": + {"project_id": \'\'\'+ str(sample['project_id']) + +\'\'\' , "name": "project1"}}' \'\'\' '''.replace('\n', '') + + command2 = ''' \'\'\'echo '{"project": + {"project_id": \'\'\'+ str(sample['project_id']) + +\'\'\' , "name": "project2"}}' \'\'\' '''.replace('\n', '') + + extra_metadata_fields_embedded = { + 'name': "project_name2", + 'host_command': command2, + 'value': "name", + } + + extra_metadata_fields = { + 'name': "project_id2", + 'host_command': command, + 'value': "project_id", + 'extra_metadata_fields': extra_metadata_fields_embedded + } + + definitions['value_attribute'] = 'project_id' + definitions['extra_metadata_fields'] = extra_metadata_fields + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = ''' + {"projects": [ + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}] + } + ''' + + def get(url, *args, **kwargs): + return return_value + + client_mock.session.get = get + manager = mock.Mock + manager._keystone = client_mock + + def discover(*args, **kwargs): + return ["https://endpoint.server.name/"] + + manager.discover = discover + samples = pollster.get_samples( + manager=manager, cache=None, + resources=["https://endpoint.server.name/"]) + + samples = list(samples) + self.assertEqual(3, len(samples)) + + self.assertEqual(samples[0].volume, 9999) + self.assertEqual(samples[1].volume, 8888) + self.assertEqual(samples[2].volume, 7777) + + self.assertEqual(samples[0].resource_metadata, + {'project_id2': 9999, + 'project_name2': 'project2'}) + self.assertEqual(samples[1].resource_metadata, + {'project_id2': 8888, + 'project_name2': 'project2'}) + self.assertEqual(samples[2].resource_metadata, + {'project_id2': 7777, + 'project_name2': 'project2'}) + + @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_xml_json_response_handler_invalid_response( self, client_mock): definitions = copy.deepcopy( @@ -410,8 +828,8 @@ class TestDynamicPollster(base.BaseTestCase): keystone_client=client_mock, resource="https://endpoint.server.name/") - xml_handling_error = logs.output[2] - json_handling_error = logs.output[3] + xml_handling_error = logs.output[3] + json_handling_error = logs.output[4] self.assertIn( 'DEBUG:ceilometer.polling.dynamic_pollster:' @@ -479,6 +897,57 @@ class TestDynamicPollster(base.BaseTestCase): resource="https://endpoint.server.name/") self.assertEqual("Mock HTTP error.", str(exception)) + def test_execute_host_command_paged_responses(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['host_command'] = ''' + echo '{"server": [{"status": "ACTIVE"}], "next": ""}' + ''' + str_json = "'{\\\"server\\\": [{\\\"status\\\": \\\"INACTIVE\\\"}]}'" + definitions['next_sample_url_attribute'] = \ + "next|\"echo \"+value+\"" + str_json + '"' + pollster = dynamic_pollster.DynamicPollster(definitions) + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples() + resp_json = [{'status': 'ACTIVE'}, {'status': 'INACTIVE'}] + self.assertEqual(resp_json, samples) + + def test_execute_host_command_response_handler(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = ['xml', 'json'] + definitions['host_command'] = 'echo "<a><y>xml\n</y><s>xml</s></a>"' + entry = 'a' + definitions['response_entries_key'] = entry + definitions.pop('url_path') + definitions.pop('endpoint_type') + pollster = dynamic_pollster.DynamicPollster(definitions) + + samples_xml = pollster.definitions.sample_gatherer. \ + execute_request_get_samples() + + definitions['host_command'] = 'echo \'{"a": {"y":"json",' \ + '\n"s":"json"}}\'' + samples_json = pollster.definitions.sample_gatherer. \ + execute_request_get_samples() + + resp_xml = {'a': {'y': 'xml', 's': 'xml'}} + resp_json = {'a': {'y': 'json', 's': 'json'}} + self.assertEqual(resp_xml[entry], samples_xml) + self.assertEqual(resp_json[entry], samples_json) + + def test_execute_host_command_invalid_command(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['host_command'] = 'invalid-command' + definitions.pop('url_path') + definitions.pop('endpoint_type') + pollster = dynamic_pollster.DynamicPollster(definitions) + + self.assertRaises( + declarative.InvalidResponseTypeException, + pollster.definitions.sample_gatherer.execute_request_get_samples) + def test_generate_new_metadata_fields_no_metadata_mapping(self): metadata = {'name': 'someName', 'value': 1} @@ -1105,7 +1574,7 @@ class TestDynamicPollster(base.BaseTestCase): sample = pollster.definitions.sample_extractor.generate_sample( pollster_sample, pollster.definitions.configurations, - manager=mock.Mock()) + manager=mock.Mock(), conf={}) self.assertEqual(1, sample.volume) self.assertEqual(2, len(sample.resource_metadata)) @@ -1127,7 +1596,7 @@ class TestDynamicPollster(base.BaseTestCase): sample = pollster.definitions.sample_extractor.generate_sample( pollster_sample, pollster.definitions.configurations, - manager=mock.Mock()) + manager=mock.Mock(), conf={}) self.assertEqual(1, sample.volume) self.assertEqual(3, len(sample.resource_metadata)) @@ -1150,7 +1619,7 @@ class TestDynamicPollster(base.BaseTestCase): sample = pollster.definitions.sample_extractor.generate_sample( pollster_sample, pollster.definitions.configurations, - manager=mock.Mock()) + manager=mock.Mock(), conf={}) self.assertEqual(1, sample.volume) self.assertEqual(3, len(sample.resource_metadata)) |