Multi metric dynamic pollsters (handling attribute values with list of objects)

The initial idea for this feature comes from the `categories` fields that we
can find in the `summary` object of the RadosGW API. Each user has a
`categories` attribute in the response; in the `categories` list, we can find
the object that presents ain a granular fashion the consumption of different
RadosGW API operations such as GET, PUT, POST, and may others.

In that context, and having in mind that we have APIs with similar data
structures, we developed an extension for the dynamic pollster that enables
multi-metric processing for a single pollster. It works as follows.

The pollster name will contain a placeholder for the variable that
identifies the "submetric". E.g. `dynamic.radosgw.api.request.{category}`.
The placeholder `{category}` indicates the object attribute that is in the list
of objects that we use to load the sub metric name. Then, we must use a special
notation in the `value_attribute` configuration to indicate that we are dealing
with a list of objects. This is achieved via `[]` (brackets); for instance, in
the `dynamic.radosgw.api.request.{category}`, we can use `[categories].ops`
as the `value_attribute`. This indicates that the value we retrieve is a list
of objects, and when the dynamic pollster processes it, we want it (the
pollster) to load the `ops` value for the sub metrics being generated

Depends-On: https://review.opendev.org/#/c/694519/
Change-Id: I6ed4632f209ac51a07687476ca316212659d72bb
Signed-off-by: Rafael Weingärtner <rafael@apache.org>

3 files changed, 555 insertions, 328 deletions

diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py
index dc7e76b1..913e30cd 100644
--- a/ceilometer/polling/dynamic_pollster.py
+++ b/ceilometer/polling/dynamic_pollster.py
@@ -17,6 +17,9 @@
     '/etc/ceilometer/pollsters.d/'. The pollster are defined in YAML files
     similar to the idea used for handling notifications.
 """
+import copy
+import re
+
 from oslo_log import log
 from oslo_utils import timeutils
 
@@ -24,7 +27,7 @@ from requests import RequestException
 
 from ceilometer import declarative
 from ceilometer.polling import plugin_base
-from ceilometer import sample
+from ceilometer import sample as ceilometer_sample
 
 from functools import reduce
 import operator
@@ -35,203 +38,447 @@ from six.moves.urllib import parse as url_parse
 LOG = log.getLogger(__name__)
 
 
-class DynamicPollster(plugin_base.PollsterBase):
+def validate_sample_type(sample_type):
+    if sample_type not in ceilometer_sample.TYPES:
+        raise declarative.DynamicPollsterDefinitionException(
+            "Invalid sample type [%s]. Valid ones are [%s]."
+            % (sample_type, ceilometer_sample.TYPES))
 
-    OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values',
-                                'value_mapping', 'default_value',
-                                'metadata_mapping',
-                                'preserve_mapped_metadata',
-                                'response_entries_key']
 
-    REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit',
-                                'value_attribute', 'endpoint_type',
-                                'url_path']
+class PollsterDefinitionBuilder(object):
 
-    # Mandatory name field
-    name = ""
+    def __init__(self, definitions):
+        self.definitions = definitions
 
-    def __init__(self, pollster_definitions, conf=None):
-        super(DynamicPollster, self).__init__(conf)
+    def build_definitions(self, configurations):
+        supported_definitions = []
+        for definition in self.definitions:
+            if definition.is_field_applicable_to_definition(configurations):
+                supported_definitions.append(definition)
 
-        self.ALL_POLLSTER_FIELDS =\
-            self.OPTIONAL_POLLSTER_FIELDS + self.REQUIRED_POLLSTER_FIELDS
+        if not supported_definitions:
+            raise declarative.DynamicPollsterDefinitionException(
+                "Your configurations do not fit any type of DynamicPollsters, "
+                "please recheck them. Used configurations = [%s]." %
+                configurations)
 
-        LOG.debug("%s instantiated with [%s]", __name__,
-                  pollster_definitions)
+        definition_name = self.join_supported_definitions_names(
+            supported_definitions)
 
-        self.pollster_definitions = pollster_definitions
-        self.validate_pollster_definition()
+        definition_parents = tuple(supported_definitions)
+        definition_attribs = {'extra_definitions': reduce(
+            lambda d1, d2: d1 + d2, map(lambda df: df.extra_definitions,
+                                        supported_definitions))}
+        definition_type = type(definition_name, definition_parents,
+                               definition_attribs)
+        return definition_type(configurations)
 
-        if 'metadata_fields' in self.pollster_definitions:
-            LOG.debug("Metadata fields configured to [%s].",
-                      self.pollster_definitions['metadata_fields'])
+    @staticmethod
+    def join_supported_definitions_names(supported_definitions):
+        return ''.join(map(lambda df: df.__name__,
+                           supported_definitions))
 
-        self.set_default_values()
 
-        self.name = self.pollster_definitions['name']
-        self.obj = self
+class PollsterSampleExtractor(object):
 
-    def set_default_values(self):
-        if 'skip_sample_values' not in self.pollster_definitions:
-            self.pollster_definitions['skip_sample_values'] = []
+    def __init__(self, definitions):
+        self.definitions = definitions
 
-        if 'value_mapping' not in self.pollster_definitions:
-            self.pollster_definitions['value_mapping'] = {}
+    def generate_new_metadata_fields(self, metadata=None,
+                                     pollster_definitions=None):
+        pollster_definitions =\
+            pollster_definitions or self.definitions.configurations
+        metadata_mapping = pollster_definitions['metadata_mapping']
+        if not metadata_mapping or not metadata:
+            return
 
-        if 'default_value' not in self.pollster_definitions:
-            self.pollster_definitions['default_value'] = -1
+        metadata_keys = list(metadata.keys())
+        for k in metadata_keys:
+            if k not in metadata_mapping:
+                continue
 
-        if 'preserve_mapped_metadata' not in self.pollster_definitions:
-            self.pollster_definitions['preserve_mapped_metadata'] = True
+            new_key = metadata_mapping[k]
+            metadata[new_key] = metadata[k]
+            LOG.debug("Generating new key [%s] with content [%s] of key [%s]",
+                      new_key, metadata[k], k)
+            if pollster_definitions['preserve_mapped_metadata']:
+                continue
 
-        if 'metadata_mapping' not in self.pollster_definitions:
-            self.pollster_definitions['metadata_mapping'] = {}
+            k_value = metadata.pop(k)
+            LOG.debug("Removed key [%s] with value [%s] from "
+                      "metadata set that is sent to Gnocchi.", k, k_value)
 
-        if 'response_entries_key' not in self.pollster_definitions:
-            self.pollster_definitions['response_entries_key'] = None
+    def generate_sample(self, pollster_sample, pollster_definitons=None):
+        pollster_definitions =\
+            pollster_definitons or self.definitions.configurations
+        metadata = []
+        if 'metadata_fields' in pollster_definitions:
+            metadata = dict((k, pollster_sample.get(k))
+                            for k in pollster_definitions['metadata_fields'])
+
+        self.generate_new_metadata_fields(
+            metadata=metadata, pollster_definitions=pollster_definitions)
+        return ceilometer_sample.Sample(
+            timestamp=timeutils.isotime(),
+            name=pollster_definitions['name'],
+            type=pollster_definitions['sample_type'],
+            unit=pollster_definitions['unit'],
+            volume=pollster_sample['value'],
+            user_id=pollster_sample.get("user_id"),
+            project_id=pollster_sample.get("project_id"),
+            resource_id=pollster_sample.get("id"),
+            resource_metadata=metadata)
+
+    def retrieve_attribute_nested_value(self, json_object,
+                                        value_attribute=None):
+
+        attribute_key = value_attribute or self.definitions.\
+            extract_attribute_key()
+        LOG.debug("Retrieving the nested keys [%s] from [%s].",
+                  attribute_key, json_object)
+        keys_and_operations = attribute_key.split("|")
+        attribute_key = keys_and_operations[0].strip()
+        nested_keys = attribute_key.split(".")
+        value = reduce(operator.getitem, nested_keys, json_object)
 
-    def validate_pollster_definition(self):
-        missing_required_fields = \
-            [field for field in self.REQUIRED_POLLSTER_FIELDS
-             if field not in self.pollster_definitions]
+        return self.operate_value(keys_and_operations, value)
+
+    def operate_value(self, keys_and_operations, value):
+        # We do not have operations to be executed against the value extracted
+        if len(keys_and_operations) < 2:
+            return value
+        for operation in keys_and_operations[1::]:
+            # The operation must be performed onto the 'value' variable
+            if 'value' not in operation:
+                raise declarative.DynamicPollsterDefinitionException(
+                    "The attribute field operation [%s] must use the ["
+                    "value] variable." % operation,
+                    self.definitions.configurations)
+            LOG.debug("Executing operation [%s] against value [%s].",
+                      operation, value)
+            value = eval(operation.strip())
+            LOG.debug("Result [%s] of operation [%s].",
+                      value, operation)
+        return value
 
-        if missing_required_fields:
-            raise declarative.DynamicPollsterDefinitionException(
-                "Required fields %s not specified."
-                % missing_required_fields, self.pollster_definitions)
 
-        sample_type = self.pollster_definitions['sample_type']
-        if sample_type not in sample.TYPES:
-            raise declarative.DynamicPollsterDefinitionException(
-                "Invalid sample type [%s]. Valid ones are [%s]."
-                % (sample_type, sample.TYPES), self.pollster_definitions)
+class SimplePollsterSampleExtractor(PollsterSampleExtractor):
 
-        for definition_key in self.pollster_definitions:
-            if definition_key not in self.ALL_POLLSTER_FIELDS:
-                LOG.warning(
-                    "Field [%s] defined in [%s] is unknown "
-                    "and will be ignored. Valid fields are [%s].",
-                    definition_key, self.pollster_definitions,
-                    self.ALL_POLLSTER_FIELDS)
+    def generate_single_sample(self, pollster_sample):
+        value = self.retrieve_attribute_nested_value(pollster_sample)
+        value = self.definitions.value_mapper.map_or_skip_value(
+            value, pollster_sample)
 
-    def get_samples(self, manager, cache, resources):
-        if not resources:
-            LOG.debug("No resources received for processing.")
-            yield None
+        if isinstance(value, SkippedSample):
+            return value
 
-        for r in resources:
-            LOG.debug("Executing get sample for resource [%s].", r)
+        pollster_sample['value'] = value
 
-            samples = list([])
-            try:
-                samples = self.execute_request_get_samples(
-                    keystone_client=manager._keystone, resource=r)
-            except RequestException as e:
-                LOG.warning("Error [%s] while loading samples for [%s] "
-                            "for dynamic pollster [%s].",
-                            e, r, self.name)
+        return self.generate_sample(pollster_sample)
+
+    def extract_sample(self, pollster_sample):
+        sample = self.generate_single_sample(pollster_sample)
+        if isinstance(sample, SkippedSample):
+            return sample
+        yield sample
 
-            for pollster_sample in samples:
-                response_value_attribute_name = self.pollster_definitions[
-                    'value_attribute']
-                value = self.retrieve_attribute_nested_value(
-                    pollster_sample, response_value_attribute_name)
-
-                skip_sample_values = \
-                    self.pollster_definitions['skip_sample_values']
-                if skip_sample_values and value in skip_sample_values:
-                    LOG.debug("Skipping sample [%s] because value [%s] "
-                              "is configured to be skipped in skip list [%s].",
-                              pollster_sample, value, skip_sample_values)
-                    continue
 
-                value = self.execute_value_mapping(value)
+class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor):
 
-                user_id = None
-                if 'user_id' in pollster_sample:
-                    user_id = pollster_sample["user_id"]
+    def extract_sample(self, pollster_sample):
+        pollster_definitions = self.definitions.configurations
+        value = self.retrieve_attribute_nested_value(pollster_sample)
+        LOG.debug("We are dealing with a multi metric pollster. The "
+                  "value we are processing is the following: [%s].",
+                  value)
 
-                project_id = None
-                if 'project_id' in pollster_sample:
-                    project_id = pollster_sample["project_id"]
+        self.validate_sample_is_list(value)
+        sub_metric_placeholder, pollster_name, sub_metric_attribute_name = \
+            self.extract_names_attrs()
 
-                resource_id = None
-                if 'id' in pollster_sample:
-                    resource_id = pollster_sample["id"]
+        value_attribute = \
+            self.extract_field_name_from_value_attribute_configuration()
+        LOG.debug("Using attribute [%s] to look for values in the "
+                  "multi metric pollster [%s] with sample [%s]",
+                  value_attribute, pollster_definitions, value)
 
-                metadata = []
-                if 'metadata_fields' in self.pollster_definitions:
-                    metadata = dict((k, pollster_sample.get(k))
-                                    for k in self.pollster_definitions[
-                                        'metadata_fields'])
-                self.generate_new_metadata_fields(metadata=metadata)
-                yield sample.Sample(
-                    timestamp=timeutils.isotime(),
+        pollster_definitions = copy.deepcopy(pollster_definitions)
+        yield from self.extract_sub_samples(value, sub_metric_attribute_name,
+                                            pollster_name, value_attribute,
+                                            sub_metric_placeholder,
+                                            pollster_definitions,
+                                            pollster_sample)
 
-                    name=self.pollster_definitions['name'],
-                    type=self.pollster_definitions['sample_type'],
-                    unit=self.pollster_definitions['unit'],
-                    volume=value,
+    def extract_sub_samples(self, value, sub_metric_attribute_name,
+                            pollster_name, value_attribute,
+                            sub_metric_placeholder, pollster_definitions,
+                            pollster_sample):
 
-                    user_id=user_id,
-                    project_id=project_id,
-                    resource_id=resource_id,
+        for sub_sample in value:
+            sub_metric_name = sub_sample[sub_metric_attribute_name]
+            new_metric_name = pollster_name.replace(
+                sub_metric_placeholder, sub_metric_name)
+            pollster_definitions['name'] = new_metric_name
 
-                    resource_metadata=metadata
-                )
+            actual_value = self.retrieve_attribute_nested_value(
+                sub_sample, value_attribute)
+
+            pollster_sample['value'] = actual_value
+
+            if self.should_skip_generate_sample(actual_value, sub_sample,
+                                                sub_metric_name):
+                continue
+
+            yield self.generate_sample(pollster_sample, pollster_definitions)
+
+    def extract_field_name_from_value_attribute_configuration(self):
+        value_attribute = self.definitions.configurations['value_attribute']
+        return self.definitions.pattern_pollster_value_attribute.match(
+            value_attribute).group(3)[1::]
+
+    def extract_names_attrs(self):
+        pollster_name = self.definitions.configurations['name']
+        sub_metric_placeholder = pollster_name.split(".").pop()
+        return (sub_metric_placeholder,
+                pollster_name,
+                self.definitions.pattern_pollster_name.match(
+                    "." + sub_metric_placeholder).group(2))
+
+    def validate_sample_is_list(self, value):
+        pollster_definitions = self.definitions.configurations
+        if not isinstance(value, list):
+            raise declarative.DynamicPollsterException(
+                "Multi metric pollster defined, but the value [%s]"
+                " obtained with [%s] attribute is not a list"
+                " of objects."
+                % (value,
+                   pollster_definitions['value_attribute']),
+                pollster_definitions)
+
+    def should_skip_generate_sample(self, actual_value, sub_sample,
+                                    sub_metric_name):
+        skip_sample_values = \
+            self.definitions.configurations['skip_sample_values']
+        if actual_value in skip_sample_values:
+            LOG.debug(
+                "Skipping multi metric sample [%s] because "
+                "value [%s] is configured to be skipped in "
+                "skip list [%s].", sub_sample, actual_value,
+                skip_sample_values)
+            return True
+        if sub_metric_name in skip_sample_values:
+            LOG.debug(
+                "Skipping sample [%s] because its sub-metric "
+                "name [%s] is configured to be skipped in "
+                "skip list [%s].", sub_sample, sub_metric_name,
+                skip_sample_values)
+            return True
+        return False
+
+
+class PollsterValueMapper(object):
+
+    def __init__(self, definitions):
+        self.definitions = definitions
+
+    def map_or_skip_value(self, value, pollster_sample):
+        skip_sample_values = \
+            self.definitions.configurations['skip_sample_values']
+
+        if value in skip_sample_values:
+            LOG.debug("Skipping sample [%s] because value [%s] "
+                      "is configured to be skipped in skip list [%s].",
+                      pollster_sample, value, skip_sample_values)
+            return SkippedSample()
+
+        return self.execute_value_mapping(value)
 
     def execute_value_mapping(self, value):
-        value_mapping = self.pollster_definitions['value_mapping']
-        if value_mapping:
-            if value in value_mapping:
-                old_value = value
-                value = value_mapping[value]
-                LOG.debug("Value mapped from [%s] to [%s]",
-                          old_value, value)
-            else:
-                default_value = \
-                    self.pollster_definitions['default_value']
-                LOG.warning(
-                    "Value [%s] was not found in value_mapping [%s]; "
-                    "therefore, we will use the default [%s].",
-                    value, value_mapping, default_value)
-                value = default_value
+        value_mapping = self.definitions.configurations['value_mapping']
+        if not value_mapping:
+            return value
+
+        if value in value_mapping:
+            old_value = value
+            value = value_mapping[value]
+            LOG.debug("Value mapped from [%s] to [%s]",
+                      old_value, value)
+        else:
+            default_value = \
+                self.definitions.configurations['default_value']
+            LOG.warning(
+                "Value [%s] was not found in value_mapping [%s]; "
+                "therefore, we will use the default [%s].",
+                value, value_mapping, default_value)
+            value = default_value
         return value
 
-    def generate_new_metadata_fields(self, metadata=None):
-        metadata_mapping = self.pollster_definitions['metadata_mapping']
-        if not metadata_mapping or not metadata:
-            return
 
-        metadata_keys = list(metadata.keys())
-        for k in metadata_keys:
-            if k not in metadata_mapping:
-                continue
+class PollsterDefinition(object):
+
+    def __init__(self, name, required=False, on_missing=lambda df: df.default,
+                 default=None, validation_regex=None, creatable=True,
+                 validator=None):
+        self.name = name
+        self.required = required
+        self.on_missing = on_missing
+        self.validation_regex = validation_regex
+        self.creatable = creatable
+        self.default = default
+        if self.validation_regex:
+            self.validation_pattern = re.compile(self.validation_regex)
+        self.validator = validator
+
+    def validate(self, val):
+        if val is None:
+            return self.on_missing(self)
+        if self.validation_regex and not self.validation_pattern.match(val):
+            raise declarative.DynamicPollsterDefinitionException(
+                "Pollster %s [%s] does not match [%s]."
+                % (self.name, val, self.validation_regex))
+
+        if self.validator:
+            self.validator(val)
+
+        return val
+
+
+class PollsterDefinitions(object):
+
+    POLLSTER_VALID_NAMES_REGEXP = "^([\w-]+)(\.[\w-]+)*(\.{[\w-]+})?$"
+
+    standard_definitions = [
+        PollsterDefinition(name='name', required=True,
+                           validation_regex=POLLSTER_VALID_NAMES_REGEXP),
+        PollsterDefinition(name='sample_type', required=True,
+                           validator=validate_sample_type),
+        PollsterDefinition(name='unit', required=True),
+        PollsterDefinition(name='endpoint_type', required=True),
+        PollsterDefinition(name='url_path', required=True),
+        PollsterDefinition(name='metadata_fields', creatable=False),
+        PollsterDefinition(name='skip_sample_values', default=[]),
+        PollsterDefinition(name='value_mapping', default={}),
+        PollsterDefinition(name='default_value', default=-1),
+        PollsterDefinition(name='metadata_mapping', default={}),
+        PollsterDefinition(name='preserve_mapped_metadata', default=True),
+        PollsterDefinition(name='response_entries_key')]
+
+    extra_definitions = []
+
+    def __init__(self, configurations):
+        self.configurations = configurations
+        self.value_mapper = PollsterValueMapper(self)
+        self.definitions = self.map_definitions()
+        self.validate_configurations(configurations)
+        self.validate_missing()
+        self.sample_gatherer = PollsterSampleGatherer(self)
+        self.sample_extractor = SimplePollsterSampleExtractor(self)
+
+    def validate_configurations(self, configurations):
+        for k, v in self.definitions.items():
+            if configurations.get(k) is not None:
+                self.configurations[k] = self.definitions[k].validate(
+                    self.configurations[k])
+            elif self.definitions[k].creatable:
+                self.configurations[k] = self.definitions[k].default
+
+    @staticmethod
+    def is_field_applicable_to_definition(configurations):
+        return True
+
+    def map_definitions(self):
+        definitions = dict(
+            map(lambda df: (df.name, df), self.standard_definitions))
+        extra_definitions = dict(
+            map(lambda df: (df.name, df), self.extra_definitions))
+        definitions.update(extra_definitions)
+        return definitions
+
+    def extract_attribute_key(self):
+        pass
+
+    def validate_missing(self):
+        required_configurations = map(lambda fdf: fdf.name,
+                                      filter(lambda df: df.required,
+                                             self.definitions.values()))
+
+        missing = list(filter(
+            lambda rf: rf not in map(lambda f: f[0],
+                                     filter(lambda f: f[1],
+                                            self.configurations.items())),
+            required_configurations))
+
+        if missing:
+            raise declarative.DynamicPollsterDefinitionException(
+                "Required fields %s not specified."
+                % missing, self.configurations)
 
-            new_key = metadata_mapping[k]
-            metadata[new_key] = metadata[k]
-            LOG.debug("Generating new key [%s] with content [%s] of key [%s]",
-                      new_key, metadata[k], k)
-            if self.pollster_definitions['preserve_mapped_metadata']:
-                continue
 
-            k_value = metadata.pop(k)
-            LOG.debug("Removed key [%s] with value [%s] from "
-                      "metadata set that is sent to Gnocchi.", k, k_value)
+class MultiMetricPollsterDefinitions(PollsterDefinitions):
+
+    MULTI_METRIC_POLLSTER_NAME_REGEXP = ".*(\.{(\w+)})$"
+    pattern_pollster_name = re.compile(
+        MULTI_METRIC_POLLSTER_NAME_REGEXP)
+    MULTI_METRIC_POLLSTER_VALUE_ATTRIBUTE_REGEXP = "^(\[(\w+)\])((\.\w+)+)$"
+    pattern_pollster_value_attribute = re.compile(
+        MULTI_METRIC_POLLSTER_VALUE_ATTRIBUTE_REGEXP)
+
+    extra_definitions = [
+        PollsterDefinition(
+            name='value_attribute', required=True,
+            validation_regex=MULTI_METRIC_POLLSTER_VALUE_ATTRIBUTE_REGEXP),
+    ]
+
+    def __init__(self, configurations):
+        super(MultiMetricPollsterDefinitions, self).__init__(configurations)
+        self.sample_extractor = MultiMetricPollsterSampleExtractor(self)
+
+    @staticmethod
+    def is_field_applicable_to_definition(configurations):
+        return configurations.get(
+            'name') and MultiMetricPollsterDefinitions.\
+            pattern_pollster_name.match(configurations['name'])
+
+    def extract_attribute_key(self):
+        return self.pattern_pollster_value_attribute.match(
+            self.configurations['value_attribute']).group(2)
+
+
+class SingleMetricPollsterDefinitions(PollsterDefinitions):
+
+    extra_definitions = [
+        PollsterDefinition(name='value_attribute', required=True)]
+
+    def __init__(self, configurations):
+        super(SingleMetricPollsterDefinitions, self).__init__(configurations)
+
+    def extract_attribute_key(self):
+        return self.configurations['value_attribute']
+
+    @staticmethod
+    def is_field_applicable_to_definition(configurations):
+        return not MultiMetricPollsterDefinitions. \
+            is_field_applicable_to_definition(configurations)
+
+
+class PollsterSampleGatherer(object):
+
+    def __init__(self, definitions):
+        self.definitions = definitions
 
     @property
     def default_discovery(self):
-        return 'endpoint:' + self.pollster_definitions['endpoint_type']
+        return 'endpoint:' + self.definitions.configurations['endpoint_type']
 
     def execute_request_get_samples(self, **kwargs):
-        resp, url = self.internal_execute_request_get_samples(kwargs)
+        resp, url = self.definitions.sample_gatherer. \
+            internal_execute_request_get_samples(kwargs)
 
         response_json = resp.json()
-
         entry_size = len(response_json)
         LOG.debug("Entries [%s] in the JSON for request [%s] "
                   "for dynamic pollster [%s].",
-                  response_json, url, self.name)
+                  response_json, url, self.definitions.configurations['name'])
 
         if entry_size > 0:
             return self.retrieve_entries_from_response(response_json)
@@ -241,7 +488,7 @@ class DynamicPollster(plugin_base.PollsterBase):
         keystone_client = kwargs['keystone_client']
         endpoint = kwargs['resource']
         url = url_parse.urljoin(
-            endpoint, self.pollster_definitions['url_path'])
+            endpoint, self.definitions.configurations['url_path'])
         resp = keystone_client.session.get(url, authenticated=True)
         if resp.status_code != requests.codes.ok:
             resp.raise_for_status()
@@ -251,41 +498,172 @@ class DynamicPollster(plugin_base.PollsterBase):
         if isinstance(response_json, list):
             return response_json
 
-        first_entry_name = self.pollster_definitions['response_entries_key']
+        first_entry_name = \
+            self.definitions.configurations['response_entries_key']
         if not first_entry_name:
             try:
                 first_entry_name = next(iter(response_json))
             except RuntimeError as e:
                 LOG.debug("Generator threw a StopIteration "
                           "and we need to catch it [%s].", e)
-        return self.retrieve_attribute_nested_value(response_json,
-                                                    first_entry_name)
+        return self.definitions.sample_extractor. \
+            retrieve_attribute_nested_value(response_json, first_entry_name)
 
-    def retrieve_attribute_nested_value(self, json_object, attribute_key):
-        LOG.debug("Retrieving the nested keys [%s] from [%s].",
-                  attribute_key, json_object)
 
-        keys_and_operations = attribute_key.split("|")
-        attribute_key = keys_and_operations[0].strip()
+class NonOpenStackApisPollsterDefinition(PollsterDefinitions):
 
-        nested_keys = attribute_key.split(".")
-        value = reduce(operator.getitem, nested_keys, json_object)
+    extra_definitions = [
+        PollsterDefinition(name='value_attribute', required=True),
+        PollsterDefinition(name='module', required=True),
+        PollsterDefinition(name='authentication_object', required=True),
+        PollsterDefinition(name='user_id_attribute'),
+        PollsterDefinition(name='resource_id_attribute'),
+        PollsterDefinition(name='barbican_secret_id', default=""),
+        PollsterDefinition(name='authentication_parameters', default=""),
+        PollsterDefinition(name='project_id_attribute'),
+        PollsterDefinition(name='endpoint_type')]
 
-        # We have operations to be executed against the value extracted
-        if len(keys_and_operations) > 1:
-            for operation in keys_and_operations[1::]:
-                # The operation must be performed onto the 'value' variable
-                if 'value' not in operation:
-                    raise declarative.DynamicPollsterDefinitionException(
-                        "The attribute field operation [%s] must use the ["
-                        "value] variable." % operation,
-                        self.pollster_definitions)
+    def __init__(self, configurations):
+        super(NonOpenStackApisPollsterDefinition, self).__init__(
+            configurations)
+        self.sample_gatherer = NonOpenStackApisSamplesGatherer(self)
 
-                LOG.debug("Executing operation [%s] against value[%s].",
-                          operation, value)
+    @staticmethod
+    def is_field_applicable_to_definition(configurations):
+        return configurations.get('module')
 
-                value = eval(operation.strip())
 
-                LOG.debug("Result [%s] of operation [%s].",
-                          value, operation)
-        return value
+class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
+
+    @property
+    def default_discovery(self):
+        return 'barbican:' + \
+               self.definitions.configurations['barbican_secret_id']
+
+    def internal_execute_request_get_samples(self, kwargs):
+        credentials = kwargs['resource']
+
+        override_credentials = self.definitions.configurations[
+            'authentication_parameters']
+        if override_credentials:
+            credentials = override_credentials
+
+        url = self.definitions.configurations['url_path']
+
+        authenticator_module_name = self.definitions.configurations['module']
+        authenticator_class_name = \
+            self.definitions.configurations['authentication_object']
+
+        imported_module = __import__(authenticator_module_name)
+        authenticator_class = getattr(imported_module,
+                                      authenticator_class_name)
+
+        authenticator_arguments = list(map(str.strip, credentials.split(",")))
+        authenticator_instance = authenticator_class(*authenticator_arguments)
+
+        resp = requests.get(
+            url,
+            auth=authenticator_instance)
+
+        if resp.status_code != requests.codes.ok:
+            raise declarative.NonOpenStackApisDynamicPollsterException(
+                "Error while executing request[%s]."
+                " Status[%s] and reason [%s]."
+                % (url, resp.status_code, resp.reason))
+
+        return resp, url
+
+    def execute_request_get_samples(self, **kwargs):
+        samples = super(NonOpenStackApisSamplesGatherer,
+                        self).execute_request_get_samples(**kwargs)
+
+        if samples:
+            user_id_attribute = self.definitions.configurations[
+                'user_id_attribute']
+            project_id_attribute = self.definitions.configurations[
+                'project_id_attribute']
+            resource_id_attribute = self.definitions.configurations[
+                'resource_id_attribute']
+
+            for request_sample in samples:
+                self.generate_new_attributes_in_sample(
+                    request_sample, user_id_attribute, 'user_id')
+                self.generate_new_attributes_in_sample(
+                    request_sample, project_id_attribute, 'project_id')
+                self.generate_new_attributes_in_sample(
+                    request_sample, resource_id_attribute, 'id')
+
+        return samples
+
+    def generate_new_attributes_in_sample(
+            self, sample, attribute_key, new_attribute_key):
+        if attribute_key:
+            attribute_value = self.definitions.sample_extractor. \
+                retrieve_attribute_nested_value(sample, attribute_key)
+
+            LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].",
+                      attribute_key, attribute_value, sample)
+
+            sample[new_attribute_key] = attribute_value
+
+
+class SkippedSample(object):
+    pass
+
+
+class DynamicPollster(plugin_base.PollsterBase):
+    # Mandatory name field
+    name = ""
+
+    def __init__(self, pollster_definitions={}, conf=None,
+                 supported_definitions=[NonOpenStackApisPollsterDefinition,
+                                        MultiMetricPollsterDefinitions,
+                                        SingleMetricPollsterDefinitions]):
+        super(DynamicPollster, self).__init__(conf)
+        self.supported_definitions = supported_definitions
+        LOG.debug("%s instantiated with [%s]", __name__,
+                  pollster_definitions)
+
+        self.definitions = PollsterDefinitionBuilder(
+            self.supported_definitions).build_definitions(pollster_definitions)
+        self.pollster_definitions = self.definitions.configurations
+        if 'metadata_fields' in self.pollster_definitions:
+            LOG.debug("Metadata fields configured to [%s].",
+                      self.pollster_definitions['metadata_fields'])
+
+        self.name = self.pollster_definitions['name']
+        self.obj = self
+
+    @property
+    def default_discovery(self):
+        return self.definitions.sample_gatherer.default_discovery()
+
+    def load_samples(self, resource, manager):
+        try:
+            return self.definitions.sample_gatherer.\
+                execute_request_get_samples(keystone_client=manager._keystone,
+                                            resource=resource)
+        except RequestException as e:
+            LOG.warning("Error [%s] while loading samples for [%s] "
+                        "for dynamic pollster [%s].",
+                        e, resource, self.name)
+
+        return list([])
+
+    def get_samples(self, manager, cache, resources):
+        if not resources:
+            LOG.debug("No resources received for processing.")
+            yield None
+
+        for r in resources:
+            LOG.debug("Executing get sample for resource [%s].", r)
+            samples = self.load_samples(r, manager)
+            for pollster_sample in samples:
+                sample = self.extract_sample(pollster_sample)
+                if isinstance(sample, SkippedSample):
+                    continue
+                yield from sample
+
+    def extract_sample(self, pollster_sample):
+        return self.definitions.sample_extractor.extract_sample(
+            pollster_sample)
diff --git a/ceilometer/polling/manager.py b/ceilometer/polling/manager.py
index 3329d16e..fbdce8b5 100644
--- a/ceilometer/polling/manager.py
+++ b/ceilometer/polling/manager.py
@@ -40,7 +40,6 @@ from ceilometer import declarative
 from ceilometer import keystone_client
 from ceilometer import messaging
 from ceilometer.polling import dynamic_pollster
-from ceilometer.polling import non_openstack_dynamic_pollster
 from ceilometer.polling import plugin_base
 from ceilometer.publisher import utils as publisher_utils
 from ceilometer import utils
@@ -340,7 +339,8 @@ class AgentManager(cotyledon.Service):
                              pollster_name, pollsters_definitions_file)
                     try:
                         pollsters_definitions[pollster_name] =\
-                            self.instantiate_dynamic_pollster(pollster_cfg)
+                            dynamic_pollster.DynamicPollster(
+                                pollster_cfg, self.conf)
                     except Exception as e:
                         LOG.error(
                             "Error [%s] while loading dynamic pollster [%s].",
@@ -355,13 +355,6 @@ class AgentManager(cotyledon.Service):
                   len(pollsters_definitions))
         return pollsters_definitions.values()
 
-    def instantiate_dynamic_pollster(self, pollster_cfg):
-        if 'module' in pollster_cfg:
-            return non_openstack_dynamic_pollster\
-                .NonOpenStackApisDynamicPollster(pollster_cfg, self.conf)
-        else:
-            return dynamic_pollster.DynamicPollster(pollster_cfg, self.conf)
-
     @staticmethod
     def _get_ext_mgr(namespace, *args, **kwargs):
         def _catch_extension_load_error(mgr, ep, exc):
diff --git a/ceilometer/polling/non_openstack_dynamic_pollster.py b/ceilometer/polling/non_openstack_dynamic_pollster.py
deleted file mode 100644
index 2854b921..00000000
--- a/ceilometer/polling/non_openstack_dynamic_pollster.py
+++ /dev/null
@@ -1,144 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-
-"""Non-OpenStack Dynamic pollster component
-    This component enables operators to create pollsters on the fly
-    via configuration for non-OpenStack APIs. This appraoch is quite
-    useful when adding metrics from APIs such as RadosGW into the Cloud
-    rating and billing modules.
-"""
-import copy
-import requests
-
-from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
-from ceilometer.polling.dynamic_pollster import DynamicPollster
-from oslo_log import log
-
-LOG = log.getLogger(__name__)
-
-
-class NonOpenStackApisDynamicPollster(DynamicPollster):
-
-    POLLSTER_REQUIRED_POLLSTER_FIELDS = ['module', 'authentication_object']
-
-    POLLSTER_OPTIONAL_POLLSTER_FIELDS = ['user_id_attribute',
-                                         'project_id_attribute',
-                                         'resource_id_attribute',
-                                         'barbican_secret_id',
-                                         'authentication_parameters'
-                                         ]
-
-    def __init__(self, pollster_definitions, conf=None):
-        # Making sure that we do not change anything in parent classes
-        self.REQUIRED_POLLSTER_FIELDS = copy.deepcopy(
-            DynamicPollster.REQUIRED_POLLSTER_FIELDS)
-        self.OPTIONAL_POLLSTER_FIELDS = copy.deepcopy(
-            DynamicPollster.OPTIONAL_POLLSTER_FIELDS)
-
-        # Non-OpenStack dynamic pollster do not need the 'endpoint_type'.
-        self.REQUIRED_POLLSTER_FIELDS.remove('endpoint_type')
-
-        self.REQUIRED_POLLSTER_FIELDS += self.POLLSTER_REQUIRED_POLLSTER_FIELDS
-        self.OPTIONAL_POLLSTER_FIELDS += self.POLLSTER_OPTIONAL_POLLSTER_FIELDS
-
-        super(NonOpenStackApisDynamicPollster, self).__init__(
-            pollster_definitions, conf)
-
-    def set_default_values(self):
-        super(NonOpenStackApisDynamicPollster, self).set_default_values()
-
-        if 'user_id_attribute' not in self.pollster_definitions:
-            self.pollster_definitions['user_id_attribute'] = None
-
-        if 'project_id_attribute' not in self.pollster_definitions:
-            self.pollster_definitions['project_id_attribute'] = None
-
-        if 'resource_id_attribute' not in self.pollster_definitions:
-            self.pollster_definitions['resource_id_attribute'] = None
-
-        if 'barbican_secret_id' not in self.pollster_definitions:
-            self.pollster_definitions['barbican_secret_id'] = ""
-
-        if 'authentication_parameters' not in self.pollster_definitions:
-            self.pollster_definitions['authentication_parameters'] = ""
-
-    @property
-    def default_discovery(self):
-        return 'barbican:' + self.pollster_definitions['barbican_secret_id']
-
-    def internal_execute_request_get_samples(self, kwargs):
-        credentials = kwargs['resource']
-
-        override_credentials = self.pollster_definitions[
-            'authentication_parameters']
-        if override_credentials:
-            credentials = override_credentials
-
-        url = self.pollster_definitions['url_path']
-
-        authenticator_module_name = self.pollster_definitions['module']
-        authenticator_class_name = \
-            self.pollster_definitions['authentication_object']
-
-        imported_module = __import__(authenticator_module_name)
-        authenticator_class = getattr(imported_module,
-                                      authenticator_class_name)
-
-        authenticator_arguments = list(map(str.strip, credentials.split(",")))
-        authenticator_instance = authenticator_class(*authenticator_arguments)
-
-        resp = requests.get(
-            url,
-            auth=authenticator_instance)
-
-        if resp.status_code != requests.codes.ok:
-            raise NonOpenStackApisDynamicPollsterException(
-                "Error while executing request[%s]."
-                " Status[%s] and reason [%s]."
-                % (url, resp.status_code, resp.reason))
-
-        return resp, url
-
-    def execute_request_get_samples(self, **kwargs):
-        samples = super(NonOpenStackApisDynamicPollster,
-                        self).execute_request_get_samples(**kwargs)
-
-        if samples:
-            user_id_attribute = self.pollster_definitions[
-                'user_id_attribute']
-            project_id_attribute = self.pollster_definitions[
-                'project_id_attribute']
-            resource_id_attribute = self.pollster_definitions[
-                'resource_id_attribute']
-
-            for sample in samples:
-                self.generate_new_attributes_in_sample(
-                    sample, user_id_attribute, 'user_id')
-                self.generate_new_attributes_in_sample(
-                    sample, project_id_attribute, 'project_id')
-                self.generate_new_attributes_in_sample(
-                    sample, resource_id_attribute, 'id')
-
-        return samples
-
-    def generate_new_attributes_in_sample(
-            self, sample, attribute_key, new_attribute_key):
-        if attribute_key:
-            attribute_value = self.retrieve_attribute_nested_value(
-                sample, attribute_key)
-
-            LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].",
-                      attribute_key, attribute_value, sample)
-
-            sample[new_attribute_key] = attribute_value