From 225f1cd7765ddb7b725c538944947ada8c52e73f Mon Sep 17 00:00:00 2001 From: Pedro Henrique Date: Mon, 18 Jul 2022 16:53:23 -0300 Subject: Add response handlers to support different response types Problem description =================== The dynamic pollsters only support APIs that produce JSON responses. Therefore the dynamic pollsters do not support APIs where the response is an XML or not Restful compliant APIs with HTTP 200 within a plain text message on errors. Proposal ======== To allow the dynamic pollsters to support other APIs response formats, we propose to add a response handling that supports multiple response types. It must be configurable in the dynamic pollsters YAML. The default continues to be JSON. Change-Id: I4886cefe06eccac2dc24adbc2fad2166bcbfdd2c --- ceilometer/declarative.py | 4 + ceilometer/polling/dynamic_pollster.py | 97 +++++++++++- .../tests/unit/polling/test_dynamic_pollster.py | 162 ++++++++++++++++++++- .../polling/test_non_openstack_dynamic_pollster.py | 6 + 4 files changed, 257 insertions(+), 12 deletions(-) (limited to 'ceilometer') diff --git a/ceilometer/declarative.py b/ceilometer/declarative.py index ed59f9a9..d30fff52 100644 --- a/ceilometer/declarative.py +++ b/ceilometer/declarative.py @@ -49,6 +49,10 @@ class DynamicPollsterDefinitionException(DynamicPollsterException): pass +class InvalidResponseTypeException(DynamicPollsterException): + pass + + class NonOpenStackApisDynamicPollsterException\ (DynamicPollsterDefinitionException): pass diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py index bb45b85f..3a37c9ee 100644 --- a/ceilometer/polling/dynamic_pollster.py +++ b/ceilometer/polling/dynamic_pollster.py @@ -18,8 +18,10 @@ similar to the idea used for handling notifications. """ import copy +import json import re import time +import xmltodict from oslo_log import log @@ -46,6 +48,80 @@ def validate_sample_type(sample_type): % (sample_type, ceilometer_sample.TYPES)) +class XMLResponseHandler(object): + """This response handler converts an XML in string format to a dict""" + + @staticmethod + def handle(response): + return xmltodict.parse(response) + + +class JsonResponseHandler(object): + """This response handler converts a JSON in string format to a dict""" + + @staticmethod + def handle(response): + return json.loads(response) + + +class PlainTextResponseHandler(object): + """This response handler converts a string to a dict {'out'=}""" + + @staticmethod + def handle(response): + return {'out': str(response)} + + +VALID_HANDLERS = { + 'json': JsonResponseHandler, + 'xml': XMLResponseHandler, + 'text': PlainTextResponseHandler +} + + +def validate_response_handler(val): + if not isinstance(val, list): + raise declarative.DynamicPollsterDefinitionException( + "Invalid response_handlers configuration. It must be a list. " + "Provided value type: %s" % type(val).__name__) + + for value in val: + if value not in VALID_HANDLERS: + raise declarative.DynamicPollsterDefinitionException( + "Invalid response_handler value [%s]. Accepted values " + "are [%s]" % (value, ', '.join(list(VALID_HANDLERS)))) + + +class ResponseHandlerChain(object): + """Tries to convert a string to a dict using the response handlers""" + + def __init__(self, response_handlers, **meta): + if not isinstance(response_handlers, list): + response_handlers = list(response_handlers) + + self.response_handlers = response_handlers + self.meta = meta + + def handle(self, response): + failed_handlers = [] + for handler in self.response_handlers: + try: + return handler.handle(response) + except Exception as e: + handler_name = handler.__name__ + failed_handlers.append(handler_name) + LOG.debug( + "Error handling response [%s] with handler [%s]: %s. " + "We will try the next one, if multiple handlers were " + "configured.", + response, handler_name, e) + + handlers_str = ', '.join(failed_handlers) + raise declarative.InvalidResponseTypeException( + "No remaining handlers to handle the response [%s], " + "used handlers [%s]. [%s]." % (response, handlers_str, self.meta)) + + class PollsterDefinitionBuilder(object): def __init__(self, definitions): @@ -440,7 +516,9 @@ class PollsterDefinitions(object): PollsterDefinition(name='timeout', default=30), PollsterDefinition(name='extra_metadata_fields_cache_seconds', default=3600), - PollsterDefinition(name='extra_metadata_fields') + PollsterDefinition(name='extra_metadata_fields'), + PollsterDefinition(name='response_handlers', default=['json'], + validator=validate_response_handler) ] extra_definitions = [] @@ -655,6 +733,11 @@ class PollsterSampleGatherer(object): def __init__(self, definitions): self.definitions = definitions + self.response_handler_chain = ResponseHandlerChain( + map(VALID_HANDLERS.get, + self.definitions.configurations['response_handlers']), + url_path=definitions.configurations['url_path'] + ) @property def default_discovery(self): @@ -668,17 +751,17 @@ class PollsterSampleGatherer(object): resp, url = self._internal_execute_request_get_samples( definitions=definitions, **kwargs) - response_json = resp.json() - entry_size = len(response_json) - LOG.debug("Entries [%s] in the JSON for request [%s] " + response_dict = self.response_handler_chain.handle(resp.text) + entry_size = len(response_dict) + LOG.debug("Entries [%s] in the DICT for request [%s] " "for dynamic pollster [%s].", - response_json, url, definitions['name']) + response_dict, url, definitions['name']) if entry_size > 0: samples = self.retrieve_entries_from_response( - response_json, definitions) + response_dict, definitions) url_to_next_sample = self.get_url_to_next_sample( - response_json, definitions) + response_dict, definitions) self.prepare_samples(definitions, samples, **kwargs) diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index e596f7b5..dfc0c576 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -14,13 +14,14 @@ """Tests for OpenStack dynamic pollster """ import copy +import json import logging from unittest import mock import requests from urllib import parse as urlparse -from ceilometer.declarative import DynamicPollsterDefinitionException +from ceilometer import declarative from ceilometer.polling import dynamic_pollster from ceilometer import sample from oslotest import base @@ -107,6 +108,11 @@ class TestDynamicPollster(base.BaseTestCase): class FakeResponse(object): status_code = None json_object = None + _text = None + + @property + def text(self): + return self._text or json.dumps(self.json_object) def json(self): return self.json_object @@ -242,9 +248,10 @@ class TestDynamicPollster(base.BaseTestCase): pollster_definition = copy.deepcopy( self.pollster_definition_only_required_fields) pollster_definition.pop(key) - exception = self.assertRaises(DynamicPollsterDefinitionException, - dynamic_pollster.DynamicPollster, - pollster_definition) + exception = self.assertRaises( + declarative.DynamicPollsterDefinitionException, + dynamic_pollster.DynamicPollster, + pollster_definition) self.assertEqual("Required fields ['%s'] not specified." % key, exception.brief_message) @@ -252,7 +259,7 @@ class TestDynamicPollster(base.BaseTestCase): self.pollster_definition_only_required_fields[ 'sample_type'] = "invalid_sample_type" exception = self.assertRaises( - DynamicPollsterDefinitionException, + declarative.DynamicPollsterDefinitionException, dynamic_pollster.DynamicPollster, self.pollster_definition_only_required_fields) self.assertEqual("Invalid sample type [invalid_sample_type]. " @@ -313,6 +320,147 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(3, len(samples)) + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_json_response_handler( + self, client_mock): + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = '{"test": [1,2,3]}' + + client_mock.session.get.return_value = return_value + + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples( + keystone_client=client_mock, + resource="https://endpoint.server.name/") + + self.assertEqual(3, len(samples)) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_xml_response_handler( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = ['xml'] + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = '123' + client_mock.session.get.return_value = return_value + + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples( + keystone_client=client_mock, + resource="https://endpoint.server.name/") + + self.assertEqual(3, len(samples)) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_xml_json_response_handler( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = ['xml', 'json'] + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = '123' + client_mock.session.get.return_value = return_value + + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples( + keystone_client=client_mock, + resource="https://endpoint.server.name/") + + self.assertEqual(3, len(samples)) + + return_value._text = '{"test": [1,2,3,4]}' + + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples( + keystone_client=client_mock, + resource="https://endpoint.server.name/") + + self.assertEqual(4, len(samples)) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_xml_json_response_handler_invalid_response( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = ['xml', 'json'] + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = 'Invalid response' + client_mock.session.get.return_value = return_value + + with self.assertLogs('ceilometer.polling.dynamic_pollster', + level='DEBUG') as logs: + gatherer = pollster.definitions.sample_gatherer + exception = self.assertRaises( + declarative.InvalidResponseTypeException, + gatherer.execute_request_get_samples, + keystone_client=client_mock, + resource="https://endpoint.server.name/") + + xml_handling_error = logs.output[2] + json_handling_error = logs.output[3] + + self.assertIn( + 'DEBUG:ceilometer.polling.dynamic_pollster:' + 'Error handling response [Invalid response] ' + 'with handler [XMLResponseHandler]', + xml_handling_error) + + self.assertIn( + 'DEBUG:ceilometer.polling.dynamic_pollster:' + 'Error handling response [Invalid response] ' + 'with handler [JsonResponseHandler]', + json_handling_error) + + self.assertEqual( + "InvalidResponseTypeException None: " + "No remaining handlers to handle the response " + "[Invalid response], used handlers " + "[XMLResponseHandler, JsonResponseHandler]. " + "[{'url_path': 'v1/test/endpoint/fake'}].", + str(exception)) + + def test_configure_response_handler_definition_invalid_value(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = ['jason'] + + exception = self.assertRaises( + declarative.DynamicPollsterDefinitionException, + dynamic_pollster.DynamicPollster, + pollster_definitions=definitions) + self.assertEqual("DynamicPollsterDefinitionException None: " + "Invalid response_handler value [jason]. " + "Accepted values are [json, xml, text]", + str(exception)) + + def test_configure_response_handler_definition_invalid_type(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = 'json' + + exception = self.assertRaises( + declarative.DynamicPollsterDefinitionException, + dynamic_pollster.DynamicPollster, + pollster_definitions=definitions) + self.assertEqual("DynamicPollsterDefinitionException None: " + "Invalid response_handlers configuration. " + "It must be a list. Provided value type: str", + str(exception)) + @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_get_samples_exception_on_request( self, client_mock): @@ -728,6 +876,10 @@ class TestDynamicPollster(base.BaseTestCase): def internal_execute_request_get_samples_mock(self, **kwargs): class Response: + @property + def text(self): + return json.dumps([sample]) + def json(self): return [sample] return Response(), "url" diff --git a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py index d8f32ff3..63c633bd 100644 --- a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py @@ -15,6 +15,7 @@ """ import copy +import json import sys from unittest import mock @@ -312,6 +313,11 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): def internal_execute_request_get_samples_mock( self, definitions, **kwargs): class Response: + + @property + def text(self): + return json.dumps([sample]) + def json(self): return [sample] return Response(), "url" -- cgit v1.2.1