Add response handlers to support different response types

Problem description
===================
The dynamic pollsters only support APIs that produce
JSON responses. Therefore the dynamic pollsters do not
support APIs where the response is an XML or not Restful
compliant APIs with HTTP 200 within a plain text message
on errors.

Proposal
========
To allow the dynamic pollsters to support other APIs
response formats, we propose to add a response handling
that supports multiple response types. It must be
configurable in the dynamic pollsters YAML. The default
continues to be JSON.

Change-Id: I4886cefe06eccac2dc24adbc2fad2166bcbfdd2c

4 files changed, 257 insertions, 12 deletions

diff --git a/ceilometer/declarative.py b/ceilometer/declarative.py
index ed59f9a9..d30fff52 100644
--- a/ceilometer/declarative.py
+++ b/ceilometer/declarative.py
@@ -49,6 +49,10 @@ class DynamicPollsterDefinitionException(DynamicPollsterException):
     pass
 
 
+class InvalidResponseTypeException(DynamicPollsterException):
+    pass
+
+
 class NonOpenStackApisDynamicPollsterException\
         (DynamicPollsterDefinitionException):
     pass
diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py
index bb45b85f..3a37c9ee 100644
--- a/ceilometer/polling/dynamic_pollster.py
+++ b/ceilometer/polling/dynamic_pollster.py
@@ -18,8 +18,10 @@
     similar to the idea used for handling notifications.
 """
 import copy
+import json
 import re
 import time
+import xmltodict
 
 from oslo_log import log
 
@@ -46,6 +48,80 @@ def validate_sample_type(sample_type):
             % (sample_type, ceilometer_sample.TYPES))
 
 
+class XMLResponseHandler(object):
+    """This response handler converts an XML in string format to a dict"""
+
+    @staticmethod
+    def handle(response):
+        return xmltodict.parse(response)
+
+
+class JsonResponseHandler(object):
+    """This response handler converts a JSON in string format to a dict"""
+
+    @staticmethod
+    def handle(response):
+        return json.loads(response)
+
+
+class PlainTextResponseHandler(object):
+    """This response handler converts a string to a dict {'out'=<string>}"""
+
+    @staticmethod
+    def handle(response):
+        return {'out': str(response)}
+
+
+VALID_HANDLERS = {
+    'json': JsonResponseHandler,
+    'xml': XMLResponseHandler,
+    'text': PlainTextResponseHandler
+}
+
+
+def validate_response_handler(val):
+    if not isinstance(val, list):
+        raise declarative.DynamicPollsterDefinitionException(
+            "Invalid response_handlers configuration. It must be a list. "
+            "Provided value type: %s" % type(val).__name__)
+
+    for value in val:
+        if value not in VALID_HANDLERS:
+            raise declarative.DynamicPollsterDefinitionException(
+                "Invalid response_handler value [%s]. Accepted values "
+                "are [%s]" % (value, ', '.join(list(VALID_HANDLERS))))
+
+
+class ResponseHandlerChain(object):
+    """Tries to convert a string to a dict using the response handlers"""
+
+    def __init__(self, response_handlers, **meta):
+        if not isinstance(response_handlers, list):
+            response_handlers = list(response_handlers)
+
+        self.response_handlers = response_handlers
+        self.meta = meta
+
+    def handle(self, response):
+        failed_handlers = []
+        for handler in self.response_handlers:
+            try:
+                return handler.handle(response)
+            except Exception as e:
+                handler_name = handler.__name__
+                failed_handlers.append(handler_name)
+                LOG.debug(
+                    "Error handling response [%s] with handler [%s]: %s. "
+                    "We will try the next one, if multiple handlers were "
+                    "configured.",
+                    response, handler_name, e)
+
+        handlers_str = ', '.join(failed_handlers)
+        raise declarative.InvalidResponseTypeException(
+            "No remaining handlers to handle the response [%s], "
+            "used handlers [%s]. [%s]." % (response, handlers_str, self.meta))
+
+
 class PollsterDefinitionBuilder(object):
 
     def __init__(self, definitions):
@@ -440,7 +516,9 @@ class PollsterDefinitions(object):
         PollsterDefinition(name='timeout', default=30),
         PollsterDefinition(name='extra_metadata_fields_cache_seconds',
                            default=3600),
-        PollsterDefinition(name='extra_metadata_fields')
+        PollsterDefinition(name='extra_metadata_fields'),
+        PollsterDefinition(name='response_handlers', default=['json'],
+                           validator=validate_response_handler)
     ]
 
     extra_definitions = []
@@ -655,6 +733,11 @@ class PollsterSampleGatherer(object):
 
     def __init__(self, definitions):
         self.definitions = definitions
+        self.response_handler_chain = ResponseHandlerChain(
+            map(VALID_HANDLERS.get,
+                self.definitions.configurations['response_handlers']),
+            url_path=definitions.configurations['url_path']
+        )
 
     @property
     def default_discovery(self):
@@ -668,17 +751,17 @@ class PollsterSampleGatherer(object):
         resp, url = self._internal_execute_request_get_samples(
             definitions=definitions, **kwargs)
 
-        response_json = resp.json()
-        entry_size = len(response_json)
-        LOG.debug("Entries [%s] in the JSON for request [%s] "
+        response_dict = self.response_handler_chain.handle(resp.text)
+        entry_size = len(response_dict)
+        LOG.debug("Entries [%s] in the DICT for request [%s] "
                   "for dynamic pollster [%s].",
-                  response_json, url, definitions['name'])
+                  response_dict, url, definitions['name'])
 
         if entry_size > 0:
             samples = self.retrieve_entries_from_response(
-                response_json, definitions)
+                response_dict, definitions)
             url_to_next_sample = self.get_url_to_next_sample(
-                response_json, definitions)
+                response_dict, definitions)
 
             self.prepare_samples(definitions, samples, **kwargs)
 
diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
index e596f7b5..dfc0c576 100644
--- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
@@ -14,13 +14,14 @@
 """Tests for OpenStack dynamic pollster
 """
 import copy
+import json
 import logging
 from unittest import mock
 
 import requests
 from urllib import parse as urlparse
 
-from ceilometer.declarative import DynamicPollsterDefinitionException
+from ceilometer import declarative
 from ceilometer.polling import dynamic_pollster
 from ceilometer import sample
 from oslotest import base
@@ -107,6 +108,11 @@ class TestDynamicPollster(base.BaseTestCase):
     class FakeResponse(object):
         status_code = None
         json_object = None
+        _text = None
+
+        @property
+        def text(self):
+            return self._text or json.dumps(self.json_object)
 
         def json(self):
             return self.json_object
@@ -242,9 +248,10 @@ class TestDynamicPollster(base.BaseTestCase):
             pollster_definition = copy.deepcopy(
                 self.pollster_definition_only_required_fields)
             pollster_definition.pop(key)
-            exception = self.assertRaises(DynamicPollsterDefinitionException,
-                                          dynamic_pollster.DynamicPollster,
-                                          pollster_definition)
+            exception = self.assertRaises(
+                declarative.DynamicPollsterDefinitionException,
+                dynamic_pollster.DynamicPollster,
+                pollster_definition)
             self.assertEqual("Required fields ['%s'] not specified."
                              % key, exception.brief_message)
 
@@ -252,7 +259,7 @@ class TestDynamicPollster(base.BaseTestCase):
         self.pollster_definition_only_required_fields[
             'sample_type'] = "invalid_sample_type"
         exception = self.assertRaises(
-            DynamicPollsterDefinitionException,
+            declarative.DynamicPollsterDefinitionException,
             dynamic_pollster.DynamicPollster,
             self.pollster_definition_only_required_fields)
         self.assertEqual("Invalid sample type [invalid_sample_type]. "
@@ -314,6 +321,147 @@ class TestDynamicPollster(base.BaseTestCase):
         self.assertEqual(3, len(samples))
 
     @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_json_response_handler(
+            self, client_mock):
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '{"test": [1,2,3]}'
+
+        client_mock.session.get.return_value = return_value
+
+        samples = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples(
+                keystone_client=client_mock,
+                resource="https://endpoint.server.name/")
+
+        self.assertEqual(3, len(samples))
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_xml_response_handler(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = ['xml']
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '<test>123</test>'
+        client_mock.session.get.return_value = return_value
+
+        samples = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples(
+                keystone_client=client_mock,
+                resource="https://endpoint.server.name/")
+
+        self.assertEqual(3, len(samples))
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_xml_json_response_handler(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = ['xml', 'json']
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '<test>123</test>'
+        client_mock.session.get.return_value = return_value
+
+        samples = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples(
+                keystone_client=client_mock,
+                resource="https://endpoint.server.name/")
+
+        self.assertEqual(3, len(samples))
+
+        return_value._text = '{"test": [1,2,3,4]}'
+
+        samples = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples(
+                keystone_client=client_mock,
+                resource="https://endpoint.server.name/")
+
+        self.assertEqual(4, len(samples))
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_xml_json_response_handler_invalid_response(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = ['xml', 'json']
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = 'Invalid response'
+        client_mock.session.get.return_value = return_value
+
+        with self.assertLogs('ceilometer.polling.dynamic_pollster',
+                             level='DEBUG') as logs:
+            gatherer = pollster.definitions.sample_gatherer
+            exception = self.assertRaises(
+                declarative.InvalidResponseTypeException,
+                gatherer.execute_request_get_samples,
+                keystone_client=client_mock,
+                resource="https://endpoint.server.name/")
+
+            xml_handling_error = logs.output[2]
+            json_handling_error = logs.output[3]
+
+            self.assertIn(
+                'DEBUG:ceilometer.polling.dynamic_pollster:'
+                'Error handling response [Invalid response] '
+                'with handler [XMLResponseHandler]',
+                xml_handling_error)
+
+            self.assertIn(
+                'DEBUG:ceilometer.polling.dynamic_pollster:'
+                'Error handling response [Invalid response] '
+                'with handler [JsonResponseHandler]',
+                json_handling_error)
+
+            self.assertEqual(
+                "InvalidResponseTypeException None: "
+                "No remaining handlers to handle the response "
+                "[Invalid response], used handlers "
+                "[XMLResponseHandler, JsonResponseHandler]. "
+                "[{'url_path': 'v1/test/endpoint/fake'}].",
+                str(exception))
+
+    def test_configure_response_handler_definition_invalid_value(self):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = ['jason']
+
+        exception = self.assertRaises(
+            declarative.DynamicPollsterDefinitionException,
+            dynamic_pollster.DynamicPollster,
+            pollster_definitions=definitions)
+        self.assertEqual("DynamicPollsterDefinitionException None: "
+                         "Invalid response_handler value [jason]. "
+                         "Accepted values are [json, xml, text]",
+                         str(exception))
+
+    def test_configure_response_handler_definition_invalid_type(self):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = 'json'
+
+        exception = self.assertRaises(
+            declarative.DynamicPollsterDefinitionException,
+            dynamic_pollster.DynamicPollster,
+            pollster_definitions=definitions)
+        self.assertEqual("DynamicPollsterDefinitionException None: "
+                         "Invalid response_handlers configuration. "
+                         "It must be a list. Provided value type: str",
+                         str(exception))
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
     def test_execute_request_get_samples_exception_on_request(
             self, client_mock):
         pollster = dynamic_pollster.DynamicPollster(
@@ -728,6 +876,10 @@ class TestDynamicPollster(base.BaseTestCase):
 
         def internal_execute_request_get_samples_mock(self, **kwargs):
             class Response:
+                @property
+                def text(self):
+                    return json.dumps([sample])
+
                 def json(self):
                     return [sample]
             return Response(), "url"
diff --git a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
index d8f32ff3..63c633bd 100644
--- a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
@@ -15,6 +15,7 @@
 """
 
 import copy
+import json
 import sys
 from unittest import mock
 
@@ -312,6 +313,11 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
         def internal_execute_request_get_samples_mock(
                 self, definitions, **kwargs):
             class Response:
+
+                @property
+                def text(self):
+                    return json.dumps([sample])
+
                 def json(self):
                     return [sample]
             return Response(), "url"