Enable OpenStack pollster to configure Ids(project, user, and resource)

This pull request enables OpenStack pollsters to configure
Ids(project, user, and resource) as the non-openstack pollster.
This is useful as some of the OpenStack APIs use other keys to represent
values such as project_id and user_id.

Change-Id: Id5ae1720ad714675722bc857a142d7bd667fe7ed

4 files changed, 148 insertions, 73 deletions

diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py
index 18a39d12..f230c833 100644
--- a/ceilometer/polling/dynamic_pollster.py
+++ b/ceilometer/polling/dynamic_pollster.py
@@ -362,7 +362,11 @@ class PollsterDefinitions(object):
         PollsterDefinition(name='metadata_mapping', default={}),
         PollsterDefinition(name='preserve_mapped_metadata', default=True),
         PollsterDefinition(name='response_entries_key'),
-        PollsterDefinition(name='next_sample_url_attribute')]
+        PollsterDefinition(name='next_sample_url_attribute'),
+        PollsterDefinition(name='user_id_attribute', default="user_id"),
+        PollsterDefinition(name='resource_id_attribute', default="id"),
+        PollsterDefinition(name='project_id_attribute', default="project_id"),
+    ]
 
     extra_definitions = []
 
@@ -482,14 +486,52 @@ class PollsterSampleGatherer(object):
                   response_json, url, self.definitions.configurations['name'])
 
         if entry_size > 0:
-            response = self.retrieve_entries_from_response(response_json)
+            samples = self.retrieve_entries_from_response(response_json)
             url_to_next_sample = self.get_url_to_next_sample(response_json)
             if url_to_next_sample:
                 kwargs['next_sample_url'] = url_to_next_sample
-                response += self.execute_request_get_samples(**kwargs)
-            return response
+                samples += self.execute_request_get_samples(**kwargs)
+
+            self.execute_id_overrides(samples)
+            return samples
         return []
 
+    def execute_id_overrides(self, samples):
+        if samples:
+            user_id_attribute = self.definitions.configurations[
+                'user_id_attribute']
+            project_id_attribute = self.definitions.configurations[
+                'project_id_attribute']
+            resource_id_attribute = self.definitions.configurations[
+                'resource_id_attribute']
+
+            for request_sample in samples:
+                self.generate_new_attributes_in_sample(
+                    request_sample, user_id_attribute, 'user_id')
+                self.generate_new_attributes_in_sample(
+                    request_sample, project_id_attribute, 'project_id')
+                self.generate_new_attributes_in_sample(
+                    request_sample, resource_id_attribute, 'id')
+
+    def generate_new_attributes_in_sample(
+            self, sample, attribute_key, new_attribute_key):
+
+        if attribute_key == new_attribute_key:
+            LOG.debug("We do not need to generate new attribute as the "
+                      "attribute_key[%s] and the new_attribute_key[%s] "
+                      "configurations are the same.",
+                      attribute_key, new_attribute_key)
+            return
+
+        if attribute_key:
+            attribute_value = self.definitions.sample_extractor.\
+                retrieve_attribute_nested_value(sample, attribute_key)
+
+            LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].",
+                      attribute_key, attribute_value, sample)
+
+            sample[new_attribute_key] = attribute_value
+
     def get_url_to_next_sample(self, resp):
         linked_sample_extractor = self.definitions.configurations[
             'next_sample_url_attribute']
@@ -545,11 +587,8 @@ class NonOpenStackApisPollsterDefinition(PollsterDefinitions):
         PollsterDefinition(name='value_attribute', required=True),
         PollsterDefinition(name='module', required=True),
         PollsterDefinition(name='authentication_object', required=True),
-        PollsterDefinition(name='user_id_attribute'),
-        PollsterDefinition(name='resource_id_attribute'),
         PollsterDefinition(name='barbican_secret_id', default=""),
         PollsterDefinition(name='authentication_parameters', default=""),
-        PollsterDefinition(name='project_id_attribute'),
         PollsterDefinition(name='endpoint_type')]
 
     def __init__(self, configurations):
@@ -602,28 +641,6 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
 
         return resp, url
 
-    def execute_request_get_samples(self, **kwargs):
-        samples = super(NonOpenStackApisSamplesGatherer,
-                        self).execute_request_get_samples(**kwargs)
-
-        if samples:
-            user_id_attribute = self.definitions.configurations[
-                'user_id_attribute']
-            project_id_attribute = self.definitions.configurations[
-                'project_id_attribute']
-            resource_id_attribute = self.definitions.configurations[
-                'resource_id_attribute']
-
-            for request_sample in samples:
-                self.generate_new_attributes_in_sample(
-                    request_sample, user_id_attribute, 'user_id')
-                self.generate_new_attributes_in_sample(
-                    request_sample, project_id_attribute, 'project_id')
-                self.generate_new_attributes_in_sample(
-                    request_sample, resource_id_attribute, 'id')
-
-        return samples
-
     def generate_new_attributes_in_sample(
             self, sample, attribute_key, new_attribute_key):
         if attribute_key:
diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
index a9c7bc01..7276cbe8 100644
--- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
@@ -714,3 +714,48 @@ class TestDynamicPollster(base.BaseTestCase):
         self.assertEqual(46, get_obj_sample.volume)
         self.assertEqual(8, list_bucket_sample.volume)
         self.assertEqual(46, put_obj_sample.volume)
+
+    def test_execute_request_get_samples_custom_ids(self):
+        sample = {'user_id_attribute': "1",
+                  'project_id_attribute': "2",
+                  'resource_id_attribute': "3",
+                  'user_id': "234",
+                  'project_id': "2334",
+                  'id': "35"}
+
+        def internal_execute_request_get_samples_mock(self, arg):
+            class Response:
+                def json(self):
+                    return [sample]
+            return Response(), "url"
+
+        original_method = dynamic_pollster.PollsterSampleGatherer.\
+            internal_execute_request_get_samples
+        try:
+            dynamic_pollster.PollsterSampleGatherer. \
+                internal_execute_request_get_samples = \
+                internal_execute_request_get_samples_mock
+
+            self.pollster_definition_all_fields[
+                'user_id_attribute'] = 'user_id_attribute'
+            self.pollster_definition_all_fields[
+                'project_id_attribute'] = 'project_id_attribute'
+            self.pollster_definition_all_fields[
+                'resource_id_attribute'] = 'resource_id_attribute'
+
+            pollster = dynamic_pollster.DynamicPollster(
+                self.pollster_definition_all_fields)
+
+            params = {"d": "d"}
+            response = pollster.definitions.sample_gatherer. \
+                execute_request_get_samples(**params)
+
+            self.assertEqual(sample['user_id_attribute'],
+                             response[0]['user_id'])
+            self.assertEqual(sample['project_id_attribute'],
+                             response[0]['project_id'])
+            self.assertEqual(sample['resource_id_attribute'],
+                             response[0]['id'])
+        finally:
+            dynamic_pollster.PollsterSampleGatherer. \
+                internal_execute_request_get_samples = original_method
diff --git a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
index 278eebb6..13767963 100644
--- a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
@@ -25,6 +25,7 @@ from ceilometer.polling.dynamic_pollster import DynamicPollster
 from ceilometer.polling.dynamic_pollster import MultiMetricPollsterDefinitions
 from ceilometer.polling.dynamic_pollster import \
     NonOpenStackApisPollsterDefinition
+from ceilometer.polling.dynamic_pollster import NonOpenStackApisSamplesGatherer
 from ceilometer.polling.dynamic_pollster import PollsterSampleGatherer
 from ceilometer.polling.dynamic_pollster import SingleMetricPollsterDefinitions
 
@@ -176,9 +177,10 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
 
         pollster_definitions = pollster.pollster_definitions
 
-        self.assertEqual(None, pollster_definitions['user_id_attribute'])
-        self.assertEqual(None, pollster_definitions['project_id_attribute'])
-        self.assertEqual(None, pollster_definitions['resource_id_attribute'])
+        self.assertEqual("user_id", pollster_definitions['user_id_attribute'])
+        self.assertEqual("project_id",
+                         pollster_definitions['project_id_attribute'])
+        self.assertEqual("id", pollster_definitions['resource_id_attribute'])
         self.assertEqual('', pollster_definitions['barbican_secret_id'])
         self.assertEqual('', pollster_definitions['authentication_parameters'])
 
@@ -300,33 +302,42 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
                   'project_id_attribute': "dfghyt432345t",
                   'resource_id_attribute': "sdfghjt543"}
 
-        def execute_request_get_samples_mock(self, **kwargs):
-            samples = [sample]
-            return samples
-
-        PollsterSampleGatherer.execute_request_get_samples = \
-            execute_request_get_samples_mock
-
-        self.pollster_definition_all_fields[
-            'user_id_attribute'] = 'user_id_attribute'
-        self.pollster_definition_all_fields[
-            'project_id_attribute'] = 'project_id_attribute'
-        self.pollster_definition_all_fields[
-            'resource_id_attribute'] = 'resource_id_attribute'
-
-        pollster = DynamicPollster(
-            self.pollster_definition_all_fields)
-
-        params = {"d": "d"}
-        response = pollster.definitions.sample_gatherer. \
-            execute_request_get_samples(**params)
-
-        self.assertEqual(sample['user_id_attribute'],
-                         response[0]['user_id'])
-        self.assertEqual(sample['project_id_attribute'],
-                         response[0]['project_id'])
-        self.assertEqual(sample['resource_id_attribute'],
-                         response[0]['id'])
+        def internal_execute_request_get_samples_mock(self, arg):
+            class Response:
+                def json(self):
+                    return [sample]
+            return Response(), "url"
+
+        original_method = NonOpenStackApisSamplesGatherer. \
+            internal_execute_request_get_samples
+        try:
+            NonOpenStackApisSamplesGatherer. \
+                internal_execute_request_get_samples = \
+                internal_execute_request_get_samples_mock
+
+            self.pollster_definition_all_fields[
+                'user_id_attribute'] = 'user_id_attribute'
+            self.pollster_definition_all_fields[
+                'project_id_attribute'] = 'project_id_attribute'
+            self.pollster_definition_all_fields[
+                'resource_id_attribute'] = 'resource_id_attribute'
+
+            pollster = DynamicPollster(
+                self.pollster_definition_all_fields)
+
+            params = {"d": "d"}
+            response = pollster.definitions.sample_gatherer. \
+                execute_request_get_samples(**params)
+
+            self.assertEqual(sample['user_id_attribute'],
+                             response[0]['user_id'])
+            self.assertEqual(sample['project_id_attribute'],
+                             response[0]['project_id'])
+            self.assertEqual(sample['resource_id_attribute'],
+                             response[0]['id'])
+        finally:
+            NonOpenStackApisSamplesGatherer. \
+                internal_execute_request_get_samples = original_method
 
     def test_execute_request_get_samples_empty_keys(self):
         sample = {'user_id_attribute': "123456789",
diff --git a/doc/source/admin/telemetry-dynamic-pollster.rst b/doc/source/admin/telemetry-dynamic-pollster.rst
index f7569fa0..fb174201 100644
--- a/doc/source/admin/telemetry-dynamic-pollster.rst
+++ b/doc/source/admin/telemetry-dynamic-pollster.rst
@@ -151,6 +151,20 @@ attributes to define a dynamic pollster:
    directly. We also accept nested values dictionaries. To use a nested value
    one can simply use ``attribute1.attribute2.<asMuchAsNeeded>.lastattribute``
 
+* ``user_id_attribute``: optional parameter. The default value is ``user_id``.
+   The name of the attribute in the entries that are processed from
+   ``response_entries_key`` elements that will be mapped to ``user_id``
+   attribute that is sent to Gnocchi.
+
+* ``project_id_attribute``: optional parameter. The default value is
+   ``project_id``. The name of the attribute in the entries that are
+   processed from ``response_entries_key`` elements that will be mapped to
+   ``project_id`` attribute that is sent to Gnocchi.
+
+* ``resource_id_attribute``: optional parameter. The default value is ``id``.
+   The name of the attribute in the entries that are processed from
+   ``response_entries_key`` elements that will be mapped to ``id`` attribute
+   that is sent to Gnocchi.
 
 The complete YAML configuration to gather data from Magnum (that has been used
 as an example) is the following:
@@ -244,8 +258,8 @@ the Dynamic pollster system. The attribute that is not supported is
 the ``endpoint_type``. The dynamic pollster system for non-OpenStack APIs
 is activated automatically when one uses the configurations ``module``.
 
-The extra parameters that are available when using the Non-OpenStack
-dynamic pollster sub-subsystem are the following:
+The extra parameters (in addition to the original ones) that are available
+when using the Non-OpenStack dynamic pollster sub-subsystem are the following:
 
 *  ``module``: required parameter. It is the python module name that Ceilometer
    has to load to use the authentication object when executing requests against
@@ -268,18 +282,6 @@ dynamic pollster sub-subsystem are the following:
   from which, Ceilometer can retrieve the comma separated values of the
   ``authentication_parameters``.
 
-* ``user_id_attribute``: optional parameter. The name of the attribute in the
-  entries that are processed from ``response_entries_key`` elements that
-  will be mapped to ``user_id`` attribute that is sent to Gnocchi.
-
-* ``project_id_attribute``: optional parameter. The name of the attribute in
-  the entries that are processed from ``response_entries_key`` elements that
-  will be mapped to ``project_id`` attribute that is sent to Gnocchi.
-
-* ``resource_id_attribute``: optional parameter. The name of the attribute
-  in the entries that are processed from ``response_entries_key`` elements that
-  will be mapped to ``id`` attribute that is sent to Gnocchi.
-
 As follows we present an example on how to convert the hard-coded pollster
 for `radosgw.api.request` metric to the dynamic pollster model: