diff options
Diffstat (limited to 'ceilometer')
-rw-r--r-- | ceilometer/declarative.py | 4 | ||||
-rw-r--r-- | ceilometer/polling/dynamic_pollster.py | 97 | ||||
-rw-r--r-- | ceilometer/tests/unit/polling/test_dynamic_pollster.py | 162 | ||||
-rw-r--r-- | ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py | 6 |
4 files changed, 257 insertions, 12 deletions
diff --git a/ceilometer/declarative.py b/ceilometer/declarative.py index ed59f9a9..d30fff52 100644 --- a/ceilometer/declarative.py +++ b/ceilometer/declarative.py @@ -49,6 +49,10 @@ class DynamicPollsterDefinitionException(DynamicPollsterException): pass +class InvalidResponseTypeException(DynamicPollsterException): + pass + + class NonOpenStackApisDynamicPollsterException\ (DynamicPollsterDefinitionException): pass diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py index bb45b85f..3a37c9ee 100644 --- a/ceilometer/polling/dynamic_pollster.py +++ b/ceilometer/polling/dynamic_pollster.py @@ -18,8 +18,10 @@ similar to the idea used for handling notifications. """ import copy +import json import re import time +import xmltodict from oslo_log import log @@ -46,6 +48,80 @@ def validate_sample_type(sample_type): % (sample_type, ceilometer_sample.TYPES)) +class XMLResponseHandler(object): + """This response handler converts an XML in string format to a dict""" + + @staticmethod + def handle(response): + return xmltodict.parse(response) + + +class JsonResponseHandler(object): + """This response handler converts a JSON in string format to a dict""" + + @staticmethod + def handle(response): + return json.loads(response) + + +class PlainTextResponseHandler(object): + """This response handler converts a string to a dict {'out'=<string>}""" + + @staticmethod + def handle(response): + return {'out': str(response)} + + +VALID_HANDLERS = { + 'json': JsonResponseHandler, + 'xml': XMLResponseHandler, + 'text': PlainTextResponseHandler +} + + +def validate_response_handler(val): + if not isinstance(val, list): + raise declarative.DynamicPollsterDefinitionException( + "Invalid response_handlers configuration. It must be a list. " + "Provided value type: %s" % type(val).__name__) + + for value in val: + if value not in VALID_HANDLERS: + raise declarative.DynamicPollsterDefinitionException( + "Invalid response_handler value [%s]. Accepted values " + "are [%s]" % (value, ', '.join(list(VALID_HANDLERS)))) + + +class ResponseHandlerChain(object): + """Tries to convert a string to a dict using the response handlers""" + + def __init__(self, response_handlers, **meta): + if not isinstance(response_handlers, list): + response_handlers = list(response_handlers) + + self.response_handlers = response_handlers + self.meta = meta + + def handle(self, response): + failed_handlers = [] + for handler in self.response_handlers: + try: + return handler.handle(response) + except Exception as e: + handler_name = handler.__name__ + failed_handlers.append(handler_name) + LOG.debug( + "Error handling response [%s] with handler [%s]: %s. " + "We will try the next one, if multiple handlers were " + "configured.", + response, handler_name, e) + + handlers_str = ', '.join(failed_handlers) + raise declarative.InvalidResponseTypeException( + "No remaining handlers to handle the response [%s], " + "used handlers [%s]. [%s]." % (response, handlers_str, self.meta)) + + class PollsterDefinitionBuilder(object): def __init__(self, definitions): @@ -440,7 +516,9 @@ class PollsterDefinitions(object): PollsterDefinition(name='timeout', default=30), PollsterDefinition(name='extra_metadata_fields_cache_seconds', default=3600), - PollsterDefinition(name='extra_metadata_fields') + PollsterDefinition(name='extra_metadata_fields'), + PollsterDefinition(name='response_handlers', default=['json'], + validator=validate_response_handler) ] extra_definitions = [] @@ -655,6 +733,11 @@ class PollsterSampleGatherer(object): def __init__(self, definitions): self.definitions = definitions + self.response_handler_chain = ResponseHandlerChain( + map(VALID_HANDLERS.get, + self.definitions.configurations['response_handlers']), + url_path=definitions.configurations['url_path'] + ) @property def default_discovery(self): @@ -668,17 +751,17 @@ class PollsterSampleGatherer(object): resp, url = self._internal_execute_request_get_samples( definitions=definitions, **kwargs) - response_json = resp.json() - entry_size = len(response_json) - LOG.debug("Entries [%s] in the JSON for request [%s] " + response_dict = self.response_handler_chain.handle(resp.text) + entry_size = len(response_dict) + LOG.debug("Entries [%s] in the DICT for request [%s] " "for dynamic pollster [%s].", - response_json, url, definitions['name']) + response_dict, url, definitions['name']) if entry_size > 0: samples = self.retrieve_entries_from_response( - response_json, definitions) + response_dict, definitions) url_to_next_sample = self.get_url_to_next_sample( - response_json, definitions) + response_dict, definitions) self.prepare_samples(definitions, samples, **kwargs) diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index e596f7b5..dfc0c576 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -14,13 +14,14 @@ """Tests for OpenStack dynamic pollster """ import copy +import json import logging from unittest import mock import requests from urllib import parse as urlparse -from ceilometer.declarative import DynamicPollsterDefinitionException +from ceilometer import declarative from ceilometer.polling import dynamic_pollster from ceilometer import sample from oslotest import base @@ -107,6 +108,11 @@ class TestDynamicPollster(base.BaseTestCase): class FakeResponse(object): status_code = None json_object = None + _text = None + + @property + def text(self): + return self._text or json.dumps(self.json_object) def json(self): return self.json_object @@ -242,9 +248,10 @@ class TestDynamicPollster(base.BaseTestCase): pollster_definition = copy.deepcopy( self.pollster_definition_only_required_fields) pollster_definition.pop(key) - exception = self.assertRaises(DynamicPollsterDefinitionException, - dynamic_pollster.DynamicPollster, - pollster_definition) + exception = self.assertRaises( + declarative.DynamicPollsterDefinitionException, + dynamic_pollster.DynamicPollster, + pollster_definition) self.assertEqual("Required fields ['%s'] not specified." % key, exception.brief_message) @@ -252,7 +259,7 @@ class TestDynamicPollster(base.BaseTestCase): self.pollster_definition_only_required_fields[ 'sample_type'] = "invalid_sample_type" exception = self.assertRaises( - DynamicPollsterDefinitionException, + declarative.DynamicPollsterDefinitionException, dynamic_pollster.DynamicPollster, self.pollster_definition_only_required_fields) self.assertEqual("Invalid sample type [invalid_sample_type]. " @@ -314,6 +321,147 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(3, len(samples)) @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_json_response_handler( + self, client_mock): + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = '{"test": [1,2,3]}' + + client_mock.session.get.return_value = return_value + + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples( + keystone_client=client_mock, + resource="https://endpoint.server.name/") + + self.assertEqual(3, len(samples)) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_xml_response_handler( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = ['xml'] + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = '<test>123</test>' + client_mock.session.get.return_value = return_value + + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples( + keystone_client=client_mock, + resource="https://endpoint.server.name/") + + self.assertEqual(3, len(samples)) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_xml_json_response_handler( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = ['xml', 'json'] + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = '<test>123</test>' + client_mock.session.get.return_value = return_value + + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples( + keystone_client=client_mock, + resource="https://endpoint.server.name/") + + self.assertEqual(3, len(samples)) + + return_value._text = '{"test": [1,2,3,4]}' + + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples( + keystone_client=client_mock, + resource="https://endpoint.server.name/") + + self.assertEqual(4, len(samples)) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_xml_json_response_handler_invalid_response( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = ['xml', 'json'] + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = 'Invalid response' + client_mock.session.get.return_value = return_value + + with self.assertLogs('ceilometer.polling.dynamic_pollster', + level='DEBUG') as logs: + gatherer = pollster.definitions.sample_gatherer + exception = self.assertRaises( + declarative.InvalidResponseTypeException, + gatherer.execute_request_get_samples, + keystone_client=client_mock, + resource="https://endpoint.server.name/") + + xml_handling_error = logs.output[2] + json_handling_error = logs.output[3] + + self.assertIn( + 'DEBUG:ceilometer.polling.dynamic_pollster:' + 'Error handling response [Invalid response] ' + 'with handler [XMLResponseHandler]', + xml_handling_error) + + self.assertIn( + 'DEBUG:ceilometer.polling.dynamic_pollster:' + 'Error handling response [Invalid response] ' + 'with handler [JsonResponseHandler]', + json_handling_error) + + self.assertEqual( + "InvalidResponseTypeException None: " + "No remaining handlers to handle the response " + "[Invalid response], used handlers " + "[XMLResponseHandler, JsonResponseHandler]. " + "[{'url_path': 'v1/test/endpoint/fake'}].", + str(exception)) + + def test_configure_response_handler_definition_invalid_value(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = ['jason'] + + exception = self.assertRaises( + declarative.DynamicPollsterDefinitionException, + dynamic_pollster.DynamicPollster, + pollster_definitions=definitions) + self.assertEqual("DynamicPollsterDefinitionException None: " + "Invalid response_handler value [jason]. " + "Accepted values are [json, xml, text]", + str(exception)) + + def test_configure_response_handler_definition_invalid_type(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = 'json' + + exception = self.assertRaises( + declarative.DynamicPollsterDefinitionException, + dynamic_pollster.DynamicPollster, + pollster_definitions=definitions) + self.assertEqual("DynamicPollsterDefinitionException None: " + "Invalid response_handlers configuration. " + "It must be a list. Provided value type: str", + str(exception)) + + @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_get_samples_exception_on_request( self, client_mock): pollster = dynamic_pollster.DynamicPollster( @@ -728,6 +876,10 @@ class TestDynamicPollster(base.BaseTestCase): def internal_execute_request_get_samples_mock(self, **kwargs): class Response: + @property + def text(self): + return json.dumps([sample]) + def json(self): return [sample] return Response(), "url" diff --git a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py index d8f32ff3..63c633bd 100644 --- a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py @@ -15,6 +15,7 @@ """ import copy +import json import sys from unittest import mock @@ -312,6 +313,11 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): def internal_execute_request_get_samples_mock( self, definitions, **kwargs): class Response: + + @property + def text(self): + return json.dumps([sample]) + def json(self): return [sample] return Response(), "url" |