Dynamic pollster system to support non-OpenStack APIs

The goal of this PR is to add the support for
non-OpenStack APIs into Ceilometer. An example of such
API is the RadosGW usage API.

Change-Id: If5e1c9bce9e2709746338e043b20d328d8fb4504

4 files changed, 244 insertions, 19 deletions

diff --git a/ceilometer/polling/discovery/non_openstack_credentials_discovery.py b/ceilometer/polling/discovery/non_openstack_credentials_discovery.py
new file mode 100644
index 00000000..956a69c7
--- /dev/null
+++ b/ceilometer/polling/discovery/non_openstack_credentials_discovery.py
@@ -0,0 +1,59 @@
+# Copyright 2014-2015 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log
+
+from ceilometer.polling.discovery.endpoint import EndpointDiscovery
+
+import six.moves.urllib.parse as urlparse
+
+import requests
+
+LOG = log.getLogger(__name__)
+
+
+class NonOpenStackCredentialsDiscovery(EndpointDiscovery):
+    """Barbican secrets discovery
+
+    Discovery that supplies non-OpenStack credentials for the dynamic
+    pollster sub-system. This solution uses the EndpointDiscovery to
+    find the Barbican URL where we can retrieve the credentials.
+    """
+
+    BARBICAN_URL_GET_PAYLOAD_PATTERN = "/v1/secrets/%s/payload"
+
+    def discover(self, manager, param=None):
+        barbican_secret = "No secrets found"
+        if not param:
+            return [barbican_secret]
+        barbican_endpoints = super(NonOpenStackCredentialsDiscovery,
+                                   self).discover("key-manager")
+        if not barbican_endpoints:
+            LOG.warning("No Barbican endpoints found to execute the"
+                        " credentials discovery process to [%s].",
+                        param)
+            return [barbican_secret]
+        else:
+            LOG.debug("Barbican endpoint found [%s].", barbican_endpoints)
+
+        barbican_server = next(iter(barbican_endpoints))
+        barbican_endpoint = self.BARBICAN_URL_GET_PAYLOAD_PATTERN % param
+        babrican_url = urlparse.urljoin(barbican_server, barbican_endpoint)
+
+        LOG.debug("Retrieving secrets from: %s.", babrican_url)
+        resp = manager._keystone.session.get(babrican_url, authenticated=True)
+        if resp.status_code != requests.codes.ok:
+            resp.raise_for_status()
+
+        return [resp._content]
diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py
index 61def7a4..f501c1b5 100644
--- a/ceilometer/polling/dynamic_pollster.py
+++ b/ceilometer/polling/dynamic_pollster.py
@@ -40,20 +40,23 @@ class DynamicPollster(plugin_base.PollsterBase):
     OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values',
                                 'value_mapping', 'default_value',
                                 'metadata_mapping',
-                                'preserve_mapped_metadata'
+                                'preserve_mapped_metadata',
                                 'response_entries_key']
 
     REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit',
                                 'value_attribute', 'endpoint_type',
                                 'url_path']
 
-    ALL_POLLSTER_FIELDS = OPTIONAL_POLLSTER_FIELDS + REQUIRED_POLLSTER_FIELDS
-
+    # Mandatory name field
     name = ""
 
     def __init__(self, pollster_definitions, conf=None):
         super(DynamicPollster, self).__init__(conf)
-        LOG.debug("Dynamic pollster created with [%s]",
+
+        self.ALL_POLLSTER_FIELDS =\
+            self.OPTIONAL_POLLSTER_FIELDS + self.REQUIRED_POLLSTER_FIELDS
+
+        LOG.debug("%s instantiated with [%s]", __name__,
                   pollster_definitions)
 
         self.pollster_definitions = pollster_definitions
@@ -63,9 +66,12 @@ class DynamicPollster(plugin_base.PollsterBase):
             LOG.debug("Metadata fields configured to [%s].",
                       self.pollster_definitions['metadata_fields'])
 
+        self.set_default_values()
+
         self.name = self.pollster_definitions['name']
         self.obj = self
 
+    def set_default_values(self):
         if 'skip_sample_values' not in self.pollster_definitions:
             self.pollster_definitions['skip_sample_values'] = []
 
@@ -113,17 +119,17 @@ class DynamicPollster(plugin_base.PollsterBase):
             LOG.debug("No resources received for processing.")
             yield None
 
-        for endpoint in resources:
-            LOG.debug("Executing get sample on URL [%s].", endpoint)
+        for r in resources:
+            LOG.debug("Executing get sample for resource [%s].", r)
 
             samples = list([])
             try:
                 samples = self.execute_request_get_samples(
-                    keystone_client=manager._keystone, endpoint=endpoint)
+                    keystone_client=manager._keystone, resource=r)
             except RequestException as e:
                 LOG.warning("Error [%s] while loading samples for [%s] "
                             "for dynamic pollster [%s].",
-                            e, endpoint, self.name)
+                            e, r, self.name)
 
             for pollster_sample in samples:
                 response_value_attribute_name = self.pollster_definitions[
@@ -149,6 +155,10 @@ class DynamicPollster(plugin_base.PollsterBase):
                 if 'project_id' in pollster_sample:
                     project_id = pollster_sample["project_id"]
 
+                resource_id = None
+                if 'id' in pollster_sample:
+                    resource_id = pollster_sample["id"]
+
                 metadata = []
                 if 'metadata_fields' in self.pollster_definitions:
                     metadata = dict((k, pollster_sample.get(k))
@@ -165,7 +175,7 @@ class DynamicPollster(plugin_base.PollsterBase):
 
                     user_id=user_id,
                     project_id=project_id,
-                    resource_id=pollster_sample["id"],
+                    resource_id=resource_id,
 
                     resource_metadata=metadata
                 )
@@ -213,12 +223,8 @@ class DynamicPollster(plugin_base.PollsterBase):
     def default_discovery(self):
         return 'endpoint:' + self.pollster_definitions['endpoint_type']
 
-    def execute_request_get_samples(self, keystone_client, endpoint):
-        url = url_parse.urljoin(
-            endpoint, self.pollster_definitions['url_path'])
-        resp = keystone_client.session.get(url, authenticated=True)
-        if resp.status_code != requests.codes.ok:
-            resp.raise_for_status()
+    def execute_request_get_samples(self, **kwargs):
+        resp, url = self.internal_execute_request_get_samples(kwargs)
 
         response_json = resp.json()
 
@@ -231,6 +237,16 @@ class DynamicPollster(plugin_base.PollsterBase):
             return self.retrieve_entries_from_response(response_json)
         return []
 
+    def internal_execute_request_get_samples(self, kwargs):
+        keystone_client = kwargs['keystone_client']
+        endpoint = kwargs['resource']
+        url = url_parse.urljoin(
+            endpoint, self.pollster_definitions['url_path'])
+        resp = keystone_client.session.get(url, authenticated=True)
+        if resp.status_code != requests.codes.ok:
+            resp.raise_for_status()
+        return resp, url
+
     def retrieve_entries_from_response(self, response_json):
         if isinstance(response_json, list):
             return response_json
diff --git a/ceilometer/polling/manager.py b/ceilometer/polling/manager.py
index 240e8b6a..3329d16e 100644
--- a/ceilometer/polling/manager.py
+++ b/ceilometer/polling/manager.py
@@ -40,6 +40,7 @@ from ceilometer import declarative
 from ceilometer import keystone_client
 from ceilometer import messaging
 from ceilometer.polling import dynamic_pollster
+from ceilometer.polling import non_openstack_dynamic_pollster
 from ceilometer.polling import plugin_base
 from ceilometer.publisher import utils as publisher_utils
 from ceilometer import utils
@@ -338,10 +339,8 @@ class AgentManager(cotyledon.Service):
                     LOG.info("Loading dynamic pollster [%s] from file [%s].",
                              pollster_name, pollsters_definitions_file)
                     try:
-                        dynamic_pollster_object = dynamic_pollster.\
-                            DynamicPollster(pollster_cfg, self.conf)
-                        pollsters_definitions[pollster_name] = \
-                            dynamic_pollster_object
+                        pollsters_definitions[pollster_name] =\
+                            self.instantiate_dynamic_pollster(pollster_cfg)
                     except Exception as e:
                         LOG.error(
                             "Error [%s] while loading dynamic pollster [%s].",
@@ -356,6 +355,13 @@ class AgentManager(cotyledon.Service):
                   len(pollsters_definitions))
         return pollsters_definitions.values()
 
+    def instantiate_dynamic_pollster(self, pollster_cfg):
+        if 'module' in pollster_cfg:
+            return non_openstack_dynamic_pollster\
+                .NonOpenStackApisDynamicPollster(pollster_cfg, self.conf)
+        else:
+            return dynamic_pollster.DynamicPollster(pollster_cfg, self.conf)
+
     @staticmethod
     def _get_ext_mgr(namespace, *args, **kwargs):
         def _catch_extension_load_error(mgr, ep, exc):
diff --git a/ceilometer/polling/non_openstack_dynamic_pollster.py b/ceilometer/polling/non_openstack_dynamic_pollster.py
new file mode 100644
index 00000000..2854b921
--- /dev/null
+++ b/ceilometer/polling/non_openstack_dynamic_pollster.py
@@ -0,0 +1,144 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+"""Non-OpenStack Dynamic pollster component
+    This component enables operators to create pollsters on the fly
+    via configuration for non-OpenStack APIs. This appraoch is quite
+    useful when adding metrics from APIs such as RadosGW into the Cloud
+    rating and billing modules.
+"""
+import copy
+import requests
+
+from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
+from ceilometer.polling.dynamic_pollster import DynamicPollster
+from oslo_log import log
+
+LOG = log.getLogger(__name__)
+
+
+class NonOpenStackApisDynamicPollster(DynamicPollster):
+
+    POLLSTER_REQUIRED_POLLSTER_FIELDS = ['module', 'authentication_object']
+
+    POLLSTER_OPTIONAL_POLLSTER_FIELDS = ['user_id_attribute',
+                                         'project_id_attribute',
+                                         'resource_id_attribute',
+                                         'barbican_secret_id',
+                                         'authentication_parameters'
+                                         ]
+
+    def __init__(self, pollster_definitions, conf=None):
+        # Making sure that we do not change anything in parent classes
+        self.REQUIRED_POLLSTER_FIELDS = copy.deepcopy(
+            DynamicPollster.REQUIRED_POLLSTER_FIELDS)
+        self.OPTIONAL_POLLSTER_FIELDS = copy.deepcopy(
+            DynamicPollster.OPTIONAL_POLLSTER_FIELDS)
+
+        # Non-OpenStack dynamic pollster do not need the 'endpoint_type'.
+        self.REQUIRED_POLLSTER_FIELDS.remove('endpoint_type')
+
+        self.REQUIRED_POLLSTER_FIELDS += self.POLLSTER_REQUIRED_POLLSTER_FIELDS
+        self.OPTIONAL_POLLSTER_FIELDS += self.POLLSTER_OPTIONAL_POLLSTER_FIELDS
+
+        super(NonOpenStackApisDynamicPollster, self).__init__(
+            pollster_definitions, conf)
+
+    def set_default_values(self):
+        super(NonOpenStackApisDynamicPollster, self).set_default_values()
+
+        if 'user_id_attribute' not in self.pollster_definitions:
+            self.pollster_definitions['user_id_attribute'] = None
+
+        if 'project_id_attribute' not in self.pollster_definitions:
+            self.pollster_definitions['project_id_attribute'] = None
+
+        if 'resource_id_attribute' not in self.pollster_definitions:
+            self.pollster_definitions['resource_id_attribute'] = None
+
+        if 'barbican_secret_id' not in self.pollster_definitions:
+            self.pollster_definitions['barbican_secret_id'] = ""
+
+        if 'authentication_parameters' not in self.pollster_definitions:
+            self.pollster_definitions['authentication_parameters'] = ""
+
+    @property
+    def default_discovery(self):
+        return 'barbican:' + self.pollster_definitions['barbican_secret_id']
+
+    def internal_execute_request_get_samples(self, kwargs):
+        credentials = kwargs['resource']
+
+        override_credentials = self.pollster_definitions[
+            'authentication_parameters']
+        if override_credentials:
+            credentials = override_credentials
+
+        url = self.pollster_definitions['url_path']
+
+        authenticator_module_name = self.pollster_definitions['module']
+        authenticator_class_name = \
+            self.pollster_definitions['authentication_object']
+
+        imported_module = __import__(authenticator_module_name)
+        authenticator_class = getattr(imported_module,
+                                      authenticator_class_name)
+
+        authenticator_arguments = list(map(str.strip, credentials.split(",")))
+        authenticator_instance = authenticator_class(*authenticator_arguments)
+
+        resp = requests.get(
+            url,
+            auth=authenticator_instance)
+
+        if resp.status_code != requests.codes.ok:
+            raise NonOpenStackApisDynamicPollsterException(
+                "Error while executing request[%s]."
+                " Status[%s] and reason [%s]."
+                % (url, resp.status_code, resp.reason))
+
+        return resp, url
+
+    def execute_request_get_samples(self, **kwargs):
+        samples = super(NonOpenStackApisDynamicPollster,
+                        self).execute_request_get_samples(**kwargs)
+
+        if samples:
+            user_id_attribute = self.pollster_definitions[
+                'user_id_attribute']
+            project_id_attribute = self.pollster_definitions[
+                'project_id_attribute']
+            resource_id_attribute = self.pollster_definitions[
+                'resource_id_attribute']
+
+            for sample in samples:
+                self.generate_new_attributes_in_sample(
+                    sample, user_id_attribute, 'user_id')
+                self.generate_new_attributes_in_sample(
+                    sample, project_id_attribute, 'project_id')
+                self.generate_new_attributes_in_sample(
+                    sample, resource_id_attribute, 'id')
+
+        return samples
+
+    def generate_new_attributes_in_sample(
+            self, sample, attribute_key, new_attribute_key):
+        if attribute_key:
+            attribute_value = self.retrieve_attribute_nested_value(
+                sample, attribute_key)
+
+            LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].",
+                      attribute_key, attribute_value, sample)
+
+            sample[new_attribute_key] = attribute_value