Merge "OpenStack Dynamic pollsters metadata enrichment with other OpenStack API's data"18.0.0.0rc118.0.0

5 files changed, 439 insertions, 132 deletions

diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py
index 53c30e42..bb45b85f 100644
--- a/ceilometer/polling/dynamic_pollster.py
+++ b/ceilometer/polling/dynamic_pollster.py
@@ -19,6 +19,7 @@
 """
 import copy
 import re
+import time
 
 from oslo_log import log
 
@@ -108,20 +109,37 @@ class PollsterSampleExtractor(object):
             LOG.debug("Removed key [%s] with value [%s] from "
                       "metadata set that is sent to Gnocchi.", k, k_value)
 
-    def generate_sample(self, pollster_sample, pollster_definitons=None):
+    def generate_sample(
+            self, pollster_sample, pollster_definitions=None, **kwargs):
+
         pollster_definitions =\
-            pollster_definitons or self.definitions.configurations
+            pollster_definitions or self.definitions.configurations
 
         metadata = dict()
         if 'metadata_fields' in pollster_definitions:
             for k in pollster_definitions['metadata_fields']:
-                val = self.retrieve_attribute_nested_value(pollster_sample, k)
+                val = self.retrieve_attribute_nested_value(
+                    pollster_sample, value_attribute=k,
+                    definitions=self.definitions.configurations)
 
                 LOG.debug("Assigning value [%s] to metadata key [%s].", val, k)
                 metadata[k] = val
 
         self.generate_new_metadata_fields(
             metadata=metadata, pollster_definitions=pollster_definitions)
+
+        extra_metadata = self.definitions.retrieve_extra_metadata(
+            kwargs['manager'], pollster_sample)
+
+        for key in extra_metadata.keys():
+            if key in metadata.keys():
+                LOG.warning("The extra metadata key [%s] already exist in "
+                            "pollster current metadata set [%s]. Therefore, "
+                            "we will ignore it with its value [%s].",
+                            key, metadata, extra_metadata[key])
+                continue
+            metadata[key] = extra_metadata[key]
+
         return ceilometer_sample.Sample(
             timestamp=ceilometer_utils.isotime(),
             name=pollster_definitions['name'],
@@ -134,15 +152,18 @@ class PollsterSampleExtractor(object):
             resource_metadata=metadata)
 
     def retrieve_attribute_nested_value(self, json_object,
-                                        value_attribute=None):
+                                        value_attribute=None,
+                                        definitions=None, **kwargs):
+        if not definitions:
+            definitions = self.definitions.configurations
 
-        attribute_key = value_attribute or self.definitions.\
-            extract_attribute_key()
+        attribute_key = value_attribute
+        if not attribute_key:
+            attribute_key = self.definitions.extract_attribute_key()
 
         LOG.debug(
             "Retrieving the nested keys [%s] from [%s] or pollster [""%s].",
-            attribute_key, json_object,
-            self.definitions.configurations["name"])
+            attribute_key, json_object, definitions["name"])
 
         keys_and_operations = attribute_key.split("|")
         attribute_key = keys_and_operations[0].strip()
@@ -153,9 +174,9 @@ class PollsterSampleExtractor(object):
             nested_keys = attribute_key.split(".")
             value = reduce(operator.getitem, nested_keys, json_object)
 
-        return self.operate_value(keys_and_operations, value)
+        return self.operate_value(keys_and_operations, value, definitions)
 
-    def operate_value(self, keys_and_operations, value):
+    def operate_value(self, keys_and_operations, value, definitions):
         # We do not have operations to be executed against the value extracted
         if len(keys_and_operations) < 2:
             return value
@@ -164,24 +185,23 @@ class PollsterSampleExtractor(object):
             if 'value' not in operation:
                 raise declarative.DynamicPollsterDefinitionException(
                     "The attribute field operation [%s] must use the ["
-                    "value] variable." % operation,
-                    self.definitions.configurations)
+                    "value] variable." % operation, definitions)
 
             LOG.debug("Executing operation [%s] against value[%s] for "
                       "pollster [%s].", operation, value,
-                      self.definitions.configurations["name"])
+                      definitions["name"])
 
             value = eval(operation.strip())
-            LOG.debug(
-                "Result [%s] of operation [%s] for pollster [%s].",
-                value, operation, self.definitions.configurations["name"])
+            LOG.debug("Result [%s] of operation [%s] for pollster [%s].",
+                      value, operation, definitions["name"])
         return value
 
 
 class SimplePollsterSampleExtractor(PollsterSampleExtractor):
 
-    def generate_single_sample(self, pollster_sample):
-        value = self.retrieve_attribute_nested_value(pollster_sample)
+    def generate_single_sample(self, pollster_sample, **kwargs):
+        value = self.retrieve_attribute_nested_value(
+            pollster_sample)
         value = self.definitions.value_mapper.map_or_skip_value(
             value, pollster_sample)
 
@@ -190,10 +210,10 @@ class SimplePollsterSampleExtractor(PollsterSampleExtractor):
 
         pollster_sample['value'] = value
 
-        return self.generate_sample(pollster_sample)
+        return self.generate_sample(pollster_sample, **kwargs)
 
-    def extract_sample(self, pollster_sample):
-        sample = self.generate_single_sample(pollster_sample)
+    def extract_sample(self, pollster_sample, **kwargs):
+        sample = self.generate_single_sample(pollster_sample, **kwargs)
         if isinstance(sample, SkippedSample):
             return sample
         yield sample
@@ -201,9 +221,11 @@ class SimplePollsterSampleExtractor(PollsterSampleExtractor):
 
 class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor):
 
-    def extract_sample(self, pollster_sample):
+    def extract_sample(self, pollster_sample, **kwargs):
         pollster_definitions = self.definitions.configurations
-        value = self.retrieve_attribute_nested_value(pollster_sample)
+
+        value = self.retrieve_attribute_nested_value(
+            pollster_sample, definitions=pollster_definitions)
         LOG.debug("We are dealing with a multi metric pollster. The "
                   "value we are processing is the following: [%s].",
                   value)
@@ -223,12 +245,12 @@ class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor):
                                             pollster_name, value_attribute,
                                             sub_metric_placeholder,
                                             pollster_definitions,
-                                            pollster_sample)
+                                            pollster_sample, **kwargs)
 
     def extract_sub_samples(self, value, sub_metric_attribute_name,
                             pollster_name, value_attribute,
                             sub_metric_placeholder, pollster_definitions,
-                            pollster_sample):
+                            pollster_sample, **kwargs):
 
         for sub_sample in value:
             sub_metric_name = sub_sample[sub_metric_attribute_name]
@@ -237,7 +259,7 @@ class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor):
             pollster_definitions['name'] = new_metric_name
 
             actual_value = self.retrieve_attribute_nested_value(
-                sub_sample, value_attribute)
+                sub_sample, value_attribute, definitions=pollster_definitions)
 
             pollster_sample['value'] = actual_value
 
@@ -245,7 +267,8 @@ class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor):
                                                 sub_metric_name):
                 continue
 
-            yield self.generate_sample(pollster_sample, pollster_definitions)
+            yield self.generate_sample(
+                pollster_sample, pollster_definitions, **kwargs)
 
     def extract_field_name_from_value_attribute_configuration(self):
         value_attribute = self.definitions.configurations['value_attribute']
@@ -392,6 +415,8 @@ class PollsterDefinitions(object):
 
     POLLSTER_VALID_NAMES_REGEXP = r"^([\w-]+)(\.[\w-]+)*(\.{[\w-]+})?$"
 
+    EXTERNAL_ENDPOINT_TYPE = "external"
+
     standard_definitions = [
         PollsterDefinition(name='name', required=True,
                            validation_regex=POLLSTER_VALID_NAMES_REGEXP),
@@ -412,7 +437,11 @@ class PollsterDefinitions(object):
         PollsterDefinition(name='resource_id_attribute', default="id"),
         PollsterDefinition(name='project_id_attribute', default="project_id"),
         PollsterDefinition(name='headers'),
-        PollsterDefinition(name='timeout', default=30)]
+        PollsterDefinition(name='timeout', default=30),
+        PollsterDefinition(name='extra_metadata_fields_cache_seconds',
+                           default=3600),
+        PollsterDefinition(name='extra_metadata_fields')
+    ]
 
     extra_definitions = []
 
@@ -424,6 +453,7 @@ class PollsterDefinitions(object):
         self.validate_missing()
         self.sample_gatherer = PollsterSampleGatherer(self)
         self.sample_extractor = SimplePollsterSampleExtractor(self)
+        self.response_cache = {}
 
     def validate_configurations(self, configurations):
         for k, v in self.definitions.items():
@@ -464,6 +494,115 @@ class PollsterDefinitions(object):
                 "Required fields %s not specified."
                 % missing, self.configurations)
 
+    def retrieve_extra_metadata(self, manager, request_sample):
+        extra_metadata_fields = self.configurations['extra_metadata_fields']
+        if extra_metadata_fields:
+            if isinstance(self, NonOpenStackApisPollsterDefinition):
+                raise declarative.NonOpenStackApisDynamicPollsterException(
+                    "Not supported the use of extra metadata gathering for "
+                    "non-openstack pollsters [%s] (yet)."
+                    % self.configurations['name'])
+
+            return self._retrieve_extra_metadata(
+                extra_metadata_fields, manager, request_sample)
+
+        LOG.debug("No extra metadata to be captured for pollsters [%s] and "
+                  "request sample [%s].", self.definitions, request_sample)
+        return {}
+
+    def _retrieve_extra_metadata(
+            self, extra_metadata_fields, manager, request_sample):
+        LOG.debug("Processing extra metadata fields [%s] for "
+                  "sample [%s].", extra_metadata_fields,
+                  request_sample)
+
+        extra_metadata_captured = {}
+        for extra_metadata in extra_metadata_fields:
+            extra_metadata_name = extra_metadata['name']
+
+            if extra_metadata_name in extra_metadata_captured.keys():
+                LOG.warning("Duplicated extra metadata name [%s]. Therefore, "
+                            "we do not process this iteration [%s].",
+                            extra_metadata_name, extra_metadata)
+                continue
+
+            LOG.debug("Processing extra metadata [%s] for sample [%s].",
+                      extra_metadata_name, request_sample)
+
+            endpoint_type = 'endpoint:' + extra_metadata['endpoint_type']
+            if not endpoint_type.endswith(
+                    PollsterDefinitions.EXTERNAL_ENDPOINT_TYPE):
+                response = self.execute_openstack_extra_metadata_gathering(
+                    endpoint_type, extra_metadata, manager, request_sample,
+                    extra_metadata_captured)
+            else:
+                raise declarative.NonOpenStackApisDynamicPollsterException(
+                    "Not supported the use of extra metadata gathering for "
+                    "non-openstack endpoints [%s] (yet)." % extra_metadata)
+
+            extra_metadata_extractor_kwargs = {
+                'value_attribute': extra_metadata['value'],
+                'sample': request_sample}
+
+            extra_metadata_value = \
+                self.sample_extractor.retrieve_attribute_nested_value(
+                    response, **extra_metadata_extractor_kwargs)
+
+            LOG.debug("Generated extra metadata [%s] with value [%s].",
+                      extra_metadata_name, extra_metadata_value)
+            extra_metadata_captured[extra_metadata_name] = extra_metadata_value
+
+        return extra_metadata_captured
+
+    def execute_openstack_extra_metadata_gathering(self, endpoint_type,
+                                                   extra_metadata, manager,
+                                                   request_sample,
+                                                   extra_metadata_captured):
+        url_for_endpoint_type = manager.discover(
+            [endpoint_type], self.response_cache)
+
+        LOG.debug("URL [%s] found for endpoint type [%s].",
+                  url_for_endpoint_type, endpoint_type)
+
+        if url_for_endpoint_type:
+            url_for_endpoint_type = url_for_endpoint_type[0]
+
+        self.sample_gatherer.generate_url_path(
+            extra_metadata, request_sample, extra_metadata_captured)
+
+        cached_response, max_ttl_for_cache = self.response_cache.get(
+            extra_metadata['url_path'], (None, None))
+
+        extra_metadata_fields_cache_seconds = extra_metadata.get(
+            'extra_metadata_fields_cache_seconds',
+            self.configurations['extra_metadata_fields_cache_seconds'])
+
+        current_time = time.time()
+        if cached_response and max_ttl_for_cache >= current_time:
+            LOG.debug("Returning response [%s] for request [%s] as the TTL "
+                      "[max=%s, current_time=%s] has not expired yet.",
+                      cached_response, extra_metadata['url_path'],
+                      max_ttl_for_cache, current_time)
+            return cached_response
+
+        if cached_response:
+            LOG.debug("Cleaning cached response [%s] for request [%s] "
+                      "as the TTL [max=%s, current_time=%s] has expired.",
+                      cached_response, extra_metadata['url_path'],
+                      max_ttl_for_cache, current_time)
+
+        response = self.sample_gatherer.execute_request_for_definitions(
+            extra_metadata, **{'manager': manager,
+                               'keystone_client': manager._keystone,
+                               'resource': url_for_endpoint_type,
+                               'execute_id_overrides': False})
+
+        max_ttl_for_cache = time.time() + extra_metadata_fields_cache_seconds
+
+        cache_tuple = (response, max_ttl_for_cache)
+        self.response_cache[extra_metadata['url_path']] = cache_tuple
+        return response
+
 
 class MultiMetricPollsterDefinitions(PollsterDefinitions):
 
@@ -522,36 +661,45 @@ class PollsterSampleGatherer(object):
         return 'endpoint:' + self.definitions.configurations['endpoint_type']
 
     def execute_request_get_samples(self, **kwargs):
-        resp, url = self.definitions.sample_gatherer. \
-            internal_execute_request_get_samples(kwargs)
+        return self.execute_request_for_definitions(
+            self.definitions.configurations, **kwargs)
+
+    def execute_request_for_definitions(self, definitions, **kwargs):
+        resp, url = self._internal_execute_request_get_samples(
+            definitions=definitions, **kwargs)
 
         response_json = resp.json()
         entry_size = len(response_json)
         LOG.debug("Entries [%s] in the JSON for request [%s] "
                   "for dynamic pollster [%s].",
-                  response_json, url, self.definitions.configurations['name'])
+                  response_json, url, definitions['name'])
 
         if entry_size > 0:
-            samples = self.retrieve_entries_from_response(response_json)
-            url_to_next_sample = self.get_url_to_next_sample(response_json)
+            samples = self.retrieve_entries_from_response(
+                response_json, definitions)
+            url_to_next_sample = self.get_url_to_next_sample(
+                response_json, definitions)
+
+            self.prepare_samples(definitions, samples, **kwargs)
+
             if url_to_next_sample:
                 kwargs['next_sample_url'] = url_to_next_sample
-                samples += self.execute_request_get_samples(**kwargs)
-
-            self.execute_id_overrides(samples)
+                samples += self.execute_request_for_definitions(
+                    definitions=definitions, **kwargs)
             return samples
         return []
 
-    def execute_id_overrides(self, samples):
-        if samples:
-            user_id_attribute = self.definitions.configurations[
-                'user_id_attribute']
-            project_id_attribute = self.definitions.configurations[
-                'project_id_attribute']
-            resource_id_attribute = self.definitions.configurations[
-                'resource_id_attribute']
-
+    def prepare_samples(
+            self, definitions, samples, execute_id_overrides=True, **kwargs):
+        if samples and execute_id_overrides:
             for request_sample in samples:
+                user_id_attribute = definitions.get(
+                    'user_id_attribute', 'user_id')
+                project_id_attribute = definitions.get(
+                    'project_id_attribute', 'project_id')
+                resource_id_attribute = definitions.get(
+                    'resource_id_attribute', 'id')
+
                 self.generate_new_attributes_in_sample(
                     request_sample, user_id_attribute, 'user_id')
                 self.generate_new_attributes_in_sample(
@@ -559,6 +707,21 @@ class PollsterSampleGatherer(object):
                 self.generate_new_attributes_in_sample(
                     request_sample, resource_id_attribute, 'id')
 
+    def generate_url_path(self, extra_metadata, sample,
+                          extra_metadata_captured):
+        if not extra_metadata.get('url_path_original'):
+            extra_metadata[
+                'url_path_original'] = extra_metadata['url_path']
+
+        extra_metadata['url_path'] = eval(
+            extra_metadata['url_path_original'])
+
+        LOG.debug("URL [%s] generated for pattern [%s] for sample [%s] and "
+                  "extra metadata captured [%s].",
+                  extra_metadata['url_path'],
+                  extra_metadata['url_path_original'], sample,
+                  extra_metadata_captured)
+
     def generate_new_attributes_in_sample(
             self, sample, attribute_key, new_attribute_key):
 
@@ -578,9 +741,9 @@ class PollsterSampleGatherer(object):
 
             sample[new_attribute_key] = attribute_value
 
-    def get_url_to_next_sample(self, resp):
-        linked_sample_extractor = self.definitions.configurations[
-            'next_sample_url_attribute']
+    def get_url_to_next_sample(self, resp, definitions):
+        linked_sample_extractor = definitions.get('next_sample_url_attribute')
+
         if not linked_sample_extractor:
             return None
 
@@ -592,37 +755,40 @@ class PollsterSampleGatherer(object):
                       "the configuration [%s]", resp, linked_sample_extractor)
         return None
 
-    def internal_execute_request_get_samples(self, kwargs):
-        keystone_client = kwargs['keystone_client']
-        url = self.get_request_linked_samples_url(kwargs)
+    def _internal_execute_request_get_samples(self, definitions=None,
+                                              keystone_client=None, **kwargs):
+        if not definitions:
+            definitions = self.definitions.configurations
 
-        request_arguments = self.create_request_arguments()
+        url = self.get_request_linked_samples_url(kwargs, definitions)
+        request_arguments = self.create_request_arguments(definitions)
         LOG.debug("Executing request against [url=%s] with parameters ["
                   "%s] for pollsters [%s]", url, request_arguments,
-                  self.definitions.configurations["name"])
-
+                  definitions["name"])
         resp = keystone_client.session.get(url, **request_arguments)
-
         if resp.status_code != requests.codes.ok:
             resp.raise_for_status()
         return resp, url
 
-    def create_request_arguments(self):
+    def create_request_arguments(self, definitions):
         request_args = {
             "authenticated": True
         }
-        request_headers = self.definitions.configurations['headers']
+        request_headers = definitions.get('headers', [])
         if request_headers:
             request_args['headers'] = request_headers
-        request_args['timeout'] = self.definitions.configurations['timeout']
+        request_args['timeout'] = definitions.get('timeout', 300)
         return request_args
 
-    def get_request_linked_samples_url(self, kwargs):
+    def get_request_linked_samples_url(self, kwargs, definitions):
         next_sample_url = kwargs.get('next_sample_url')
         if next_sample_url:
             return self.get_next_page_url(kwargs, next_sample_url)
+
+        LOG.debug("Generating url with [%s] and path [%s].",
+                  kwargs, definitions['url_path'])
         return self.get_request_url(
-            kwargs, self.definitions.configurations['url_path'])
+            kwargs, definitions['url_path'])
 
     def get_next_page_url(self, kwargs, next_sample_url):
         parse_result = urlparse.urlparse(next_sample_url)
@@ -634,19 +800,19 @@ class PollsterSampleGatherer(object):
         endpoint = kwargs['resource']
         return urlparse.urljoin(endpoint, url_path)
 
-    def retrieve_entries_from_response(self, response_json):
+    def retrieve_entries_from_response(self, response_json, definitions):
         if isinstance(response_json, list):
             return response_json
 
-        first_entry_name = \
-            self.definitions.configurations['response_entries_key']
+        first_entry_name = definitions.get('response_entries_key')
+
         if not first_entry_name:
             try:
                 first_entry_name = next(iter(response_json))
             except RuntimeError as e:
                 LOG.debug("Generator threw a StopIteration "
                           "and we need to catch it [%s].", e)
-        return self.definitions.sample_extractor. \
+        return self.definitions.sample_extractor.\
             retrieve_attribute_nested_value(response_json, first_entry_name)
 
 
@@ -677,19 +843,17 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
         return 'barbican:' + \
                self.definitions.configurations['barbican_secret_id']
 
-    def internal_execute_request_get_samples(self, kwargs):
+    def _internal_execute_request_get_samples(self, definitions, **kwargs):
         credentials = kwargs['resource']
 
-        override_credentials = self.definitions.configurations[
-            'authentication_parameters']
+        override_credentials = definitions['authentication_parameters']
         if override_credentials:
             credentials = override_credentials
 
-        url = self.get_request_linked_samples_url(kwargs)
+        url = self.get_request_linked_samples_url(kwargs, definitions)
 
-        authenticator_module_name = self.definitions.configurations['module']
-        authenticator_class_name = \
-            self.definitions.configurations['authentication_object']
+        authenticator_module_name = definitions['module']
+        authenticator_class_name = definitions['authentication_object']
 
         imported_module = __import__(authenticator_module_name)
         authenticator_class = getattr(imported_module,
@@ -698,12 +862,12 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
         authenticator_arguments = list(map(str.strip, credentials.split(",")))
         authenticator_instance = authenticator_class(*authenticator_arguments)
 
-        request_arguments = self.create_request_arguments()
+        request_arguments = self.create_request_arguments(definitions)
         request_arguments["auth"] = authenticator_instance
 
         LOG.debug("Executing request against [url=%s] with parameters ["
                   "%s] for pollsters [%s]", url, request_arguments,
-                  self.definitions.configurations["name"])
+                  definitions["name"])
         resp = requests.get(url, **request_arguments)
 
         if resp.status_code != requests.codes.ok:
@@ -714,9 +878,10 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
 
         return resp, url
 
-    def create_request_arguments(self):
+    def create_request_arguments(self, definitions):
         request_arguments = super(
-            NonOpenStackApisSamplesGatherer, self).create_request_arguments()
+            NonOpenStackApisSamplesGatherer, self).create_request_arguments(
+            definitions)
 
         request_arguments.pop("authenticated")
 
@@ -728,28 +893,6 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
             return url_path
         return urlparse.urljoin(endpoint, url_path)
 
-    def execute_request_get_samples(self, **kwargs):
-        samples = super(NonOpenStackApisSamplesGatherer,
-                        self).execute_request_get_samples(**kwargs)
-
-        if samples:
-            user_id_attribute = self.definitions.configurations[
-                'user_id_attribute']
-            project_id_attribute = self.definitions.configurations[
-                'project_id_attribute']
-            resource_id_attribute = self.definitions.configurations[
-                'resource_id_attribute']
-
-            for request_sample in samples:
-                self.generate_new_attributes_in_sample(
-                    request_sample, user_id_attribute, 'user_id')
-                self.generate_new_attributes_in_sample(
-                    request_sample, project_id_attribute, 'project_id')
-                self.generate_new_attributes_in_sample(
-                    request_sample, resource_id_attribute, 'id')
-
-        return samples
-
     def generate_new_attributes_in_sample(
             self, sample, attribute_key, new_attribute_key):
         if attribute_key:
@@ -796,8 +939,9 @@ class DynamicPollster(plugin_base.PollsterBase):
     def load_samples(self, resource, manager):
         try:
             return self.definitions.sample_gatherer.\
-                execute_request_get_samples(keystone_client=manager._keystone,
-                                            resource=resource)
+                execute_request_get_samples(manager=manager,
+                                            resource=resource,
+                                            keystone_client=manager._keystone)
         except RequestException as e:
             LOG.warning("Error [%s] while loading samples for [%s] "
                         "for dynamic pollster [%s].",
@@ -814,11 +958,12 @@ class DynamicPollster(plugin_base.PollsterBase):
             LOG.debug("Executing get sample for resource [%s].", r)
             samples = self.load_samples(r, manager)
             for pollster_sample in samples:
-                sample = self.extract_sample(pollster_sample)
+                kwargs = {'manager': manager, 'resource': r}
+                sample = self.extract_sample(pollster_sample, **kwargs)
                 if isinstance(sample, SkippedSample):
                     continue
                 yield from sample
 
-    def extract_sample(self, pollster_sample):
+    def extract_sample(self, pollster_sample, **kwargs):
         return self.definitions.sample_extractor.extract_sample(
-            pollster_sample)
+            pollster_sample, **kwargs)
diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
index fa5d8bae..e596f7b5 100644
--- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
@@ -463,7 +463,7 @@ class TestDynamicPollster(base.BaseTestCase):
 
         self.assertEqual(0, len(samples_list))
 
-    def fake_sample_list(self, keystone_client=None, resource=None):
+    def fake_sample_list(self, **kwargs):
         samples_list = list()
         samples_list.append(
             {'name': "sample5", 'volume': 5, 'description': "desc-sample-5",
@@ -520,7 +520,7 @@ class TestDynamicPollster(base.BaseTestCase):
 
         response = [{"object1-attr1": 1}, {"object1-attr2": 2}]
         entries = pollster.definitions.sample_gatherer. \
-            retrieve_entries_from_response(response)
+            retrieve_entries_from_response(response, pollster.definitions)
 
         self.assertEqual(response, entries)
 
@@ -538,8 +538,9 @@ class TestDynamicPollster(base.BaseTestCase):
 
         response = {"first": first_entries_from_response,
                     "second": second_entries_from_response}
-        entries = pollster.definitions.sample_gatherer. \
-            retrieve_entries_from_response(response)
+        entries = pollster.definitions.sample_gatherer.\
+            retrieve_entries_from_response(
+                response, pollster.definitions.configurations)
 
         self.assertEqual(first_entries_from_response, entries)
 
@@ -557,7 +558,8 @@ class TestDynamicPollster(base.BaseTestCase):
         response = {"first": first_entries_from_response,
                     "second": second_entries_from_response}
         entries = pollster.definitions.sample_gatherer. \
-            retrieve_entries_from_response(response)
+            retrieve_entries_from_response(response,
+                                           pollster.definitions.configurations)
 
         self.assertEqual(second_entries_from_response, entries)
 
@@ -640,7 +642,7 @@ class TestDynamicPollster(base.BaseTestCase):
 
         self.assertEqual(expected_value_after_operations, returned_value)
 
-    def fake_sample_multi_metric(self, keystone_client=None, resource=None):
+    def fake_sample_multi_metric(self, **kwargs):
         multi_metric_sample_list = [
             {"categories": [
                 {
@@ -724,17 +726,17 @@ class TestDynamicPollster(base.BaseTestCase):
                   'project_id': "2334",
                   'id': "35"}
 
-        def internal_execute_request_get_samples_mock(self, arg):
+        def internal_execute_request_get_samples_mock(self, **kwargs):
             class Response:
                 def json(self):
                     return [sample]
             return Response(), "url"
 
         original_method = dynamic_pollster.PollsterSampleGatherer.\
-            internal_execute_request_get_samples
+            _internal_execute_request_get_samples
         try:
             dynamic_pollster.PollsterSampleGatherer. \
-                internal_execute_request_get_samples = \
+                _internal_execute_request_get_samples = \
                 internal_execute_request_get_samples_mock
 
             self.pollster_definition_all_fields[
@@ -759,7 +761,7 @@ class TestDynamicPollster(base.BaseTestCase):
                              response[0]['id'])
         finally:
             dynamic_pollster.PollsterSampleGatherer. \
-                internal_execute_request_get_samples = original_method
+                _internal_execute_request_get_samples = original_method
 
     def test_retrieve_attribute_self_reference_sample(self):
         key = " . | value['key1']['subKey1'][0]['d'] if 'key1' in value else 0"
@@ -795,7 +797,7 @@ class TestDynamicPollster(base.BaseTestCase):
         pollster = dynamic_pollster.DynamicPollster(pollster_definition)
 
         request_args = pollster.definitions.sample_gatherer\
-            .create_request_arguments()
+            .create_request_arguments(pollster.definitions.configurations)
 
         self.assertTrue("headers" in request_args)
         self.assertEqual(2, len(request_args["headers"]))
@@ -821,7 +823,7 @@ class TestDynamicPollster(base.BaseTestCase):
         pollster = dynamic_pollster.DynamicPollster(pollster_definition)
 
         request_args = pollster.definitions.sample_gatherer\
-            .create_request_arguments()
+            .create_request_arguments(pollster.definitions.configurations)
 
         self.assertTrue("headers" in request_args)
         self.assertTrue("authenticated" in request_args)
@@ -843,7 +845,8 @@ class TestDynamicPollster(base.BaseTestCase):
             self.pollster_definition_only_required_fields)
 
         request_args =\
-            pollster.definitions.sample_gatherer.create_request_arguments()
+            pollster.definitions.sample_gatherer.create_request_arguments(
+                pollster.definitions.configurations)
 
         self.assertTrue("headers" not in request_args)
         self.assertTrue("authenticated" in request_args)
@@ -902,7 +905,8 @@ class TestDynamicPollster(base.BaseTestCase):
 
         kwargs = {'resource': base_url}
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(
+                kwargs, pollster.definitions.configurations)
 
         self.assertEqual(expected_url, url)
 
@@ -917,7 +921,7 @@ class TestDynamicPollster(base.BaseTestCase):
                   'next_sample_url': expected_url}
 
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(kwargs, pollster.definitions)
 
         self.assertEqual(expected_url, url)
 
@@ -932,7 +936,8 @@ class TestDynamicPollster(base.BaseTestCase):
                   'next_sample_url': "/next_page"}
 
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(
+                kwargs, pollster.definitions.configurations)
 
         self.assertEqual(expected_url, url)
 
@@ -947,7 +952,8 @@ class TestDynamicPollster(base.BaseTestCase):
                            'value': 1}
 
         sample = pollster.definitions.sample_extractor.generate_sample(
-            pollster_sample)
+            pollster_sample, pollster.definitions.configurations,
+            manager=mock.Mock())
 
         self.assertEqual(1, sample.volume)
         self.assertEqual(2, len(sample.resource_metadata))
@@ -968,7 +974,8 @@ class TestDynamicPollster(base.BaseTestCase):
                            'value': 1}
 
         sample = pollster.definitions.sample_extractor.generate_sample(
-            pollster_sample)
+            pollster_sample, pollster.definitions.configurations,
+            manager=mock.Mock())
 
         self.assertEqual(1, sample.volume)
         self.assertEqual(3, len(sample.resource_metadata))
@@ -990,7 +997,8 @@ class TestDynamicPollster(base.BaseTestCase):
                            'value': 1}
 
         sample = pollster.definitions.sample_extractor.generate_sample(
-            pollster_sample)
+            pollster_sample, pollster.definitions.configurations,
+            manager=mock.Mock())
 
         self.assertEqual(1, sample.volume)
         self.assertEqual(3, len(sample.resource_metadata))
diff --git a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
index d9ba2205..d8f32ff3 100644
--- a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
@@ -47,7 +47,7 @@ OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values',
 ALL_POLLSTER_FIELDS = REQUIRED_POLLSTER_FIELDS + OPTIONAL_POLLSTER_FIELDS
 
 
-def fake_sample_multi_metric(self, keystone_client=None, resource=None):
+def fake_sample_multi_metric(self, **kwargs):
     multi_metric_sample_list = [
         {"user_id": "UID-U007",
          "project_id": "UID-P007",
@@ -236,8 +236,9 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
 
         kwargs = {'resource': "credentials"}
 
-        resp, url = pollster.definitions.sample_gatherer. \
-            internal_execute_request_get_samples(kwargs)
+        resp, url = pollster.definitions.sample_gatherer.\
+            _internal_execute_request_get_samples(
+                pollster.definitions.configurations, **kwargs)
 
         self.assertEqual(
             self.pollster_definition_only_required_fields['url_path'], url)
@@ -265,7 +266,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
                 exception = self.assertRaises(
                     NonOpenStackApisDynamicPollsterException,
                     pollster.definitions.sample_gatherer.
-                    internal_execute_request_get_samples, kwargs)
+                    _internal_execute_request_get_samples,
+                    pollster.definitions.configurations, **kwargs)
 
                 self.assertEqual(
                     "NonOpenStackApisDynamicPollsterException"
@@ -307,17 +309,18 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
                   'project_id_attribute': "dfghyt432345t",
                   'resource_id_attribute': "sdfghjt543"}
 
-        def internal_execute_request_get_samples_mock(self, arg):
+        def internal_execute_request_get_samples_mock(
+                self, definitions, **kwargs):
             class Response:
                 def json(self):
                     return [sample]
             return Response(), "url"
 
         original_method = NonOpenStackApisSamplesGatherer. \
-            internal_execute_request_get_samples
+            _internal_execute_request_get_samples
         try:
             NonOpenStackApisSamplesGatherer. \
-                internal_execute_request_get_samples = \
+                _internal_execute_request_get_samples = \
                 internal_execute_request_get_samples_mock
 
             self.pollster_definition_all_fields[
@@ -342,7 +345,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
                              response[0]['id'])
         finally:
             NonOpenStackApisSamplesGatherer. \
-                internal_execute_request_get_samples = original_method
+                _internal_execute_request_get_samples = original_method
 
     def test_execute_request_get_samples_empty_keys(self):
         sample = {'user_id_attribute': "123456789",
@@ -446,7 +449,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
 
         kwargs = {'resource': "non-openstack-resource"}
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(
+                kwargs, pollster.definitions.configurations)
 
         self.assertEqual(expected_url, url)
 
@@ -461,7 +465,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
         kwargs = {'next_sample_url': expected_url}
 
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(kwargs, pollster.definitions)
 
         self.assertEqual(expected_url, url)
 
@@ -476,6 +480,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
         kwargs = {'next_sample_url': next_sample_path}
 
         url = pollster.definitions.sample_gatherer\
-            .get_request_linked_samples_url(kwargs)
+            .get_request_linked_samples_url(
+                kwargs, pollster.definitions.configurations)
 
         self.assertEqual(expected_url, url)
diff --git a/doc/source/admin/telemetry-dynamic-pollster.rst b/doc/source/admin/telemetry-dynamic-pollster.rst
index 9a321cb6..d861c07f 100644
--- a/doc/source/admin/telemetry-dynamic-pollster.rst
+++ b/doc/source/admin/telemetry-dynamic-pollster.rst
@@ -746,3 +746,148 @@ presented as follows:
         url_path: "v1/test-volumes"
         response_entries_key: "servers"
         next_sample_url_attribute: "server_link | filter(lambda v: v.get('rel') == 'next', value) | list(value) | value[0] | value.get('href')"
+
+OpenStack Dynamic pollsters metadata enrichment with other OpenStack API's data
+-------------------------------------------------------------------------------
+
+Sometimes we want/need to add/gather extra metadata for the samples being
+handled by Ceilometer Dynamic pollsters, such as the project name, domain id,
+domain name, and other metadata that are not always accessible via the
+OpenStack component where the sample is gathered.
+
+For instance, when gathering the status of virtual machines (VMs) from Nova,
+we only have the `tenant_id`, which must be used as the `project_id`. However,
+for billing and later invoicing one might need/want the project name, domain
+id, and other metadata that are available in Keystone (and maybe some others
+that are scattered over other components). To achieve that, one can use the
+OpenStack metadata enrichment option. This feature is only available
+to *OpenStack pollsters*, and can only gather extra metadata from OpenStack
+APIs. As follows we present an example that shows a dynamic pollster
+configuration to gather virtual machine (VM) status, and to enrich the data
+pushed to the storage backend (e.g. Gnocchi) with project name, domain ID,
+and domain name.
+
+    .. code-block:: yaml
+
+      ---
+
+      - name: "dynamic_pollster.instance.status"
+        next_sample_url_attribute: "server_links | filter(lambda v: v.get('rel') == 'next', value) | list(value) | value[0] | value.get('href') | value.replace('http:', 'https:')"
+        sample_type: "gauge"
+        unit: "server"
+        value_attribute: "status"
+        endpoint_type: "compute"
+        url_path: "/v2.1/servers/detail?all_tenants=true"
+        headers:
+          "Openstack-API-Version": "compute 2.65"
+        project_id_attribute: "tenant_id"
+        metadata_fields:
+          - "status"
+          - "name"
+          - "flavor.vcpus"
+          - "flavor.ram"
+          - "flavor.disk"
+          - "flavor.ephemeral"
+          - "flavor.swap"
+          - "flavor.original_name"
+          - "image | value or { 'id': '' } | value['id']"
+          - "OS-EXT-AZ:availability_zone"
+          - "OS-EXT-SRV-ATTR:host"
+          - "user_id"
+          - "tags | ','.join(value)"
+          - "locked"
+        value_mapping:
+          ACTIVE: "1"
+        default_value: 0
+        metadata_mapping:
+          "OS-EXT-AZ:availability_zone": "dynamic_availability_zone"
+          "OS-EXT-SRV-ATTR:host": "dynamic_host"
+          "flavor.original_name": "dynamic_flavor_name"
+          "flavor.vcpus": "dynamic_flavor_vcpus"
+          "flavor.ram": "dynamic_flavor_ram"
+          "flavor.disk": "dynamic_flavor_disk"
+          "flavor.ephemeral": "dynamic_flavor_ephemeral"
+          "flavor.swap": "dynamic_flavor_swap"
+          "image | value or { 'id': '' } | value['id']": "dynamic_image_ref"
+          "name": "dynamic_display_name"
+          "locked": "dynamic_locked"
+          "tags | ','.join(value)": "dynamic_tags"
+        extra_metadata_fields_cache_seconds: 3600
+        extra_metadata_fields:
+          - name: "project_name"
+            endpoint_type: "identity"
+            url_path: "'/v3/projects/' + str(sample['project_id'])"
+            headers:
+              "Openstack-API-Version": "identity latest"
+            value: "name"
+            extra_metadata_fields_cache_seconds: 1800 # overriding the default cache policy
+          - name: "domain_id"
+            endpoint_type: "identity"
+            url_path: "'/v3/projects/' + str(sample['project_id'])"
+            headers:
+              "Openstack-API-Version": "identity latest"
+            value: "domain_id"
+          - name: "domain_name"
+            endpoint_type: "identity"
+            url_path: "'/v3/domains/' + str(extra_metadata_captured['domain_id'])"
+            headers:
+              "Openstack-API-Version": "identity latest"
+            value: "name"
+
+The above example can be used to gather and persist in the backend the
+status of VMs. It will persist `1` in the backend as a measure for every
+collecting period if the VM's status is `ACTIVE`, and `0` otherwise. This is
+quite useful to create hashmap rating rules for running VMs in CloudKitty.
+Then, to enrich the resource in the storage backend, we are adding extra
+metadata that are collected in Keystone via the `extra_metadata_fields`
+options.
+
+The metadata enrichment feature has the following options:
+
+*  ``extra_metadata_fields_cache_seconds``: optional parameter. Defines the
+   extra metadata request's response cache. Some requests, such as the ones
+   executed against Keystone to retrieve extra metadata are rather static.
+   Therefore, one does not need to constantly re-execute the request. That
+   is the reason why we cache the response of such requests. By default the
+   cache time to live (TTL) for responses is `3600` seconds. However, this
+   value can be increased of decreased.
+
+*  ``extra_metadata_fields``: optional parameter. This option is a list of
+   objects, where each one of its elements is an extra metadata definition.
+   Each one of the extra metadata definition can have the options defined in
+   the dynamic pollsters such as to handle paged responses, operations on the
+   extracted values, headers and so on. The basic options that must be
+   defined for an extra metadata definitions are the following:
+
+    *  ``name``:  This option is mandatory. The name of the extra metadata.
+       This is the name that is going to be used by the metadata. If there is
+       already any other metadata gathered via `metadata_fields` option or
+       transformed via `metadata_mapping` configuration, this metadata is
+       going to be discarded.
+
+    *  ``endpoint_type``:  The endpoint type that we want to execute the
+       call against. This option is mandatory. It works similarly to the
+       `endpoint_type` option in the dynamic pollster definition.
+
+    *  ``url_path``: This option is mandatory. It works similarly to the
+       `url_path` option in the dynamic pollster definition. However, this
+       `one enables operators to execute/evaluate expressions in runtime, which
+       `allows one to retrieve the information from previously gathered
+       metadata via ``extra_metadata_captured` dictionary, or via the
+       `sample` itself.
+
+    *  ``value``: This configuration is mandatory. It works similarly to the
+       `value_attribute` option in the dynamic pollster definition. It is
+       the value we want to extract from the response, and assign in the
+       metadata being generated.
+
+    *  ``headers``: This option is optional. It works similarly to the
+       `headers` option in the dynamic pollster definition.
+
+    *  ``next_sample_url_attribute``: This option is optional. It works
+       similarly to the `next_sample_url_attribute` option in the dynamic
+       pollster definition.
+
+    *  ``response_entries_key``: This option is optional. It works
+       similarly to the `response_entries_key` option in the dynamic
+       pollster definition.
diff --git a/releasenotes/notes/openstack-dynamic-pollsters-metadata-enrichment-703cf5914cf0c578.yaml b/releasenotes/notes/openstack-dynamic-pollsters-metadata-enrichment-703cf5914cf0c578.yaml
new file mode 100644
index 00000000..27c040b7
--- /dev/null
+++ b/releasenotes/notes/openstack-dynamic-pollsters-metadata-enrichment-703cf5914cf0c578.yaml
@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    OpenStack Dynamic pollsters metadata enrichment with other OpenStack API's data.