OpenStack Dynamic pollsters metadata enrichment with other OpenStack API's data

Sometimes we want/need to add/gather extra metadata for the samples
being handled by Ceilometer Dynamic pollsters, such as the project
name, domain id, domain name, and other metadata that are not always
accessible via the OpenStack component where the sample is gathered.

For instance, when gathering the status of virtual machines (VMs) from
Nova, we only have the tenant_id, which must be used as the project_id.
However, for billing and later invoicing one might need/want the
project name, domain id, and other metadata that are available in
Keystone (and maybe some others that are scattered over other
components). To achieve that, one can use the OpenStack metadata
enrichment option. This feature is only available to OpenStack
pollsters, and can only gather extra metadata from OpenStack APIs.

This patch introduces a new option in the OpenStack Dynamic pollsters,
that enable operators to enrich sample's metadata with information that
can be captured in other OpenStack APIs such as Keystone, Nova, Neutron,
and many others.

Change-Id: I079bf26cf26fcbcf678dba10469516a1dcb52c0d

2 files changed, 43 insertions, 30 deletions

diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
index fa5d8bae..e596f7b5 100644
--- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
@@ -463,7 +463,7 @@ class TestDynamicPollster(base.BaseTestCase):
 
         self.assertEqual(0, len(samples_list))
 
-    def fake_sample_list(self, keystone_client=None, resource=None):
+    def fake_sample_list(self, **kwargs):
         samples_list = list()
         samples_list.append(
             {'name': "sample5", 'volume': 5, 'description': "desc-sample-5",
@@ -520,7 +520,7 @@ class TestDynamicPollster(base.BaseTestCase):
 
         response = [{"object1-attr1": 1}, {"object1-attr2": 2}]
         entries = pollster.definitions.sample_gatherer. \
-            retrieve_entries_from_response(response)
+            retrieve_entries_from_response(response, pollster.definitions)
 
         self.assertEqual(response, entries)
 
@@ -538,8 +538,9 @@ class TestDynamicPollster(base.BaseTestCase):
 
         response = {"first": first_entries_from_response,
                     "second": second_entries_from_response}
-        entries = pollster.definitions.sample_gatherer. \
-            retrieve_entries_from_response(response)
+        entries = pollster.definitions.sample_gatherer.\
+            retrieve_entries_from_response(
+                response, pollster.definitions.configurations)
 
         self.assertEqual(first_entries_from_response, entries)
 
@@ -557,7 +558,8 @@ class TestDynamicPollster(base.BaseTestCase):
         response = {"first": first_entries_from_response,
                     "second": second_entries_from_response}
         entries = pollster.definitions.sample_gatherer. \
-            retrieve_entries_from_response(response)
+            retrieve_entries_from_response(response,
+                                           pollster.definitions.configurations)
 
         self.assertEqual(second_entries_from_response, entries)
 
@@ -640,7 +642,7 @@ class TestDynamicPollster(base.BaseTestCase):
 
         self.assertEqual(expected_value_after_operations, returned_value)
 
-    def fake_sample_multi_metric(self, keystone_client=None, resource=None):
+    def fake_sample_multi_metric(self, **kwargs):
         multi_metric_sample_list = [
             {"categories": [
                 {
@@ -724,17 +726,17 @@ class TestDynamicPollster(base.BaseTestCase):
                   'project_id': "2334",
                   'id': "35"}
 
-        def internal_execute_request_get_samples_mock(self, arg):
+        def internal_execute_request_get_samples_mock(self, **kwargs):
             class Response:
                 def json(self):
                     return [sample]
             return Response(), "url"
 
         original_method = dynamic_pollster.PollsterSampleGatherer.\
-            internal_execute_request_get_samples
+            _internal_execute_request_get_samples
         try:
             dynamic_pollster.PollsterSampleGatherer. \
-                internal_execute_request_get_samples = \
+                _internal_execute_request_get_samples = \
                 internal_execute_request_get_samples_mock
 
             self.pollster_definition_all_fields[
@@ -759,7 +761,7 @@ class TestDynamicPollster(base.BaseTestCase):
                              response[0]['id'])
         finally:
             dynamic_pollster.PollsterSampleGatherer. \
-                internal_execute_request_get_samples = original_method
+                _internal_execute_request_get_samples = original_method
 
     def test_retrieve_attribute_self_reference_sample(self):
         key = " . | value['key1']['subKey1'][0]['d'] if 'key1' in value else 0"
@@ -795,7 +797,7 @@ class TestDynamicPollster(base.BaseTestCase):
         pollster = dynamic_pollster.DynamicPollster(pollster_definition)
 
         request_args = pollster.definitions.sample_gatherer\
-            .create_request_arguments()
+            .create_request_arguments(pollster.definitions.configurations)
 
         self.assertTrue("headers" in request_args)
         self.assertEqual(2, len(request_args["headers"]))
@@ -821,7 +823,7 @@ class TestDynamicPollster(base.BaseTestCase):
         pollster = dynamic_pollster.DynamicPollster(pollster_definition)
 
         request_args = pollster.definitions.sample_gatherer\
-            .create_request_arguments()
+            .create_request_arguments(pollster.definitions.configurations)
 
         self.assertTrue("headers" in request_args)
         self.assertTrue("authenticated" in request_args)
@@ -843,7 +845,8 @@ class TestDynamicPollster(base.BaseTestCase):
             self.pollster_definition_only_required_fields)
 
         request_args =\
-            pollster.definitions.sample_gatherer.create_request_arguments()
+            pollster.definitions.sample_gatherer.create_request_arguments(
+                pollster.definitions.configurations)
 
         self.assertTrue("headers" not in request_args)
         self.assertTrue("authenticated" in request_args)
@@ -902,7 +905,8 @@ class TestDynamicPollster(base.BaseTestCase):
 
         kwargs = {'resource': base_url}
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(
+                kwargs, pollster.definitions.configurations)
 
         self.assertEqual(expected_url, url)
 
@@ -917,7 +921,7 @@ class TestDynamicPollster(base.BaseTestCase):
                   'next_sample_url': expected_url}
 
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(kwargs, pollster.definitions)
 
         self.assertEqual(expected_url, url)
 
@@ -932,7 +936,8 @@ class TestDynamicPollster(base.BaseTestCase):
                   'next_sample_url': "/next_page"}
 
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(
+                kwargs, pollster.definitions.configurations)
 
         self.assertEqual(expected_url, url)
 
@@ -947,7 +952,8 @@ class TestDynamicPollster(base.BaseTestCase):
                            'value': 1}
 
         sample = pollster.definitions.sample_extractor.generate_sample(
-            pollster_sample)
+            pollster_sample, pollster.definitions.configurations,
+            manager=mock.Mock())
 
         self.assertEqual(1, sample.volume)
         self.assertEqual(2, len(sample.resource_metadata))
@@ -968,7 +974,8 @@ class TestDynamicPollster(base.BaseTestCase):
                            'value': 1}
 
         sample = pollster.definitions.sample_extractor.generate_sample(
-            pollster_sample)
+            pollster_sample, pollster.definitions.configurations,
+            manager=mock.Mock())
 
         self.assertEqual(1, sample.volume)
         self.assertEqual(3, len(sample.resource_metadata))
@@ -990,7 +997,8 @@ class TestDynamicPollster(base.BaseTestCase):
                            'value': 1}
 
         sample = pollster.definitions.sample_extractor.generate_sample(
-            pollster_sample)
+            pollster_sample, pollster.definitions.configurations,
+            manager=mock.Mock())
 
         self.assertEqual(1, sample.volume)
         self.assertEqual(3, len(sample.resource_metadata))
diff --git a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
index d9ba2205..d8f32ff3 100644
--- a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
@@ -47,7 +47,7 @@ OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values',
 ALL_POLLSTER_FIELDS = REQUIRED_POLLSTER_FIELDS + OPTIONAL_POLLSTER_FIELDS
 
 
-def fake_sample_multi_metric(self, keystone_client=None, resource=None):
+def fake_sample_multi_metric(self, **kwargs):
     multi_metric_sample_list = [
         {"user_id": "UID-U007",
          "project_id": "UID-P007",
@@ -236,8 +236,9 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
 
         kwargs = {'resource': "credentials"}
 
-        resp, url = pollster.definitions.sample_gatherer. \
-            internal_execute_request_get_samples(kwargs)
+        resp, url = pollster.definitions.sample_gatherer.\
+            _internal_execute_request_get_samples(
+                pollster.definitions.configurations, **kwargs)
 
         self.assertEqual(
             self.pollster_definition_only_required_fields['url_path'], url)
@@ -265,7 +266,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
                 exception = self.assertRaises(
                     NonOpenStackApisDynamicPollsterException,
                     pollster.definitions.sample_gatherer.
-                    internal_execute_request_get_samples, kwargs)
+                    _internal_execute_request_get_samples,
+                    pollster.definitions.configurations, **kwargs)
 
                 self.assertEqual(
                     "NonOpenStackApisDynamicPollsterException"
@@ -307,17 +309,18 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
                   'project_id_attribute': "dfghyt432345t",
                   'resource_id_attribute': "sdfghjt543"}
 
-        def internal_execute_request_get_samples_mock(self, arg):
+        def internal_execute_request_get_samples_mock(
+                self, definitions, **kwargs):
             class Response:
                 def json(self):
                     return [sample]
             return Response(), "url"
 
         original_method = NonOpenStackApisSamplesGatherer. \
-            internal_execute_request_get_samples
+            _internal_execute_request_get_samples
         try:
             NonOpenStackApisSamplesGatherer. \
-                internal_execute_request_get_samples = \
+                _internal_execute_request_get_samples = \
                 internal_execute_request_get_samples_mock
 
             self.pollster_definition_all_fields[
@@ -342,7 +345,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
                              response[0]['id'])
         finally:
             NonOpenStackApisSamplesGatherer. \
-                internal_execute_request_get_samples = original_method
+                _internal_execute_request_get_samples = original_method
 
     def test_execute_request_get_samples_empty_keys(self):
         sample = {'user_id_attribute': "123456789",
@@ -446,7 +449,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
 
         kwargs = {'resource': "non-openstack-resource"}
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(
+                kwargs, pollster.definitions.configurations)
 
         self.assertEqual(expected_url, url)
 
@@ -461,7 +465,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
         kwargs = {'next_sample_url': expected_url}
 
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(kwargs, pollster.definitions)
 
         self.assertEqual(expected_url, url)
 
@@ -476,6 +480,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
         kwargs = {'next_sample_url': next_sample_path}
 
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(
+                kwargs, pollster.definitions.configurations)
 
         self.assertEqual(expected_url, url)