Add support to host command dynamic pollster definitions

Problem description
===================
Today we have some hardcoded pollsters that are gathering
data from running virtual machines through libvirt or
different programs running in the compute nodes. However,
the Dynamic pollster definition does not support this kind of
operations to gather data, it only supports HTTP Rest
requests to collect data. Therefore, it is not possible to
use the dynamic pollster definition to create a YML based
pollster that runs and collects data from Libvirt in the
compute nodes.

Proposal
========
To allow host commands/scripts in the Dynamic pollsters,
we propose to add a new pollster definition using the
`os.subprocess` lib to run host commands to collect
Host/VMs data and store them in the configured backend.
This will provide more flexibility and make the
Dynamic pollsters able to be used in Ceilometer compute
 instances as well.

Change-Id: I50b8dc341ce457780416b41d138e35f5a0d083b6
Depends-On: https://review.opendev.org/c/openstack/ceilometer/+/850253

2 files changed, 654 insertions, 128 deletions

diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py
index 3a37c9ee..c42b1848 100644
--- a/ceilometer/polling/dynamic_pollster.py
+++ b/ceilometer/polling/dynamic_pollster.py
@@ -20,6 +20,7 @@
 import copy
 import json
 import re
+import subprocess
 import time
 import xmltodict
 
@@ -205,7 +206,7 @@ class PollsterSampleExtractor(object):
             metadata=metadata, pollster_definitions=pollster_definitions)
 
         extra_metadata = self.definitions.retrieve_extra_metadata(
-            kwargs['manager'], pollster_sample)
+            kwargs['manager'], pollster_sample, kwargs['conf'])
 
         for key in extra_metadata.keys():
             if key in metadata.keys():
@@ -518,7 +519,8 @@ class PollsterDefinitions(object):
                            default=3600),
         PollsterDefinition(name='extra_metadata_fields'),
         PollsterDefinition(name='response_handlers', default=['json'],
-                           validator=validate_response_handler)
+                           validator=validate_response_handler),
+        PollsterDefinition(name='base_metadata', default={})
     ]
 
     extra_definitions = []
@@ -572,114 +574,75 @@ class PollsterDefinitions(object):
                 "Required fields %s not specified."
                 % missing, self.configurations)
 
-    def retrieve_extra_metadata(self, manager, request_sample):
+    def retrieve_extra_metadata(self, manager, request_sample, pollster_conf):
         extra_metadata_fields = self.configurations['extra_metadata_fields']
         if extra_metadata_fields:
-            if isinstance(self, NonOpenStackApisPollsterDefinition):
-                raise declarative.NonOpenStackApisDynamicPollsterException(
-                    "Not supported the use of extra metadata gathering for "
-                    "non-openstack pollsters [%s] (yet)."
-                    % self.configurations['name'])
-
-            return self._retrieve_extra_metadata(
-                extra_metadata_fields, manager, request_sample)
+            extra_metadata_samples = {}
+            extra_metadata_by_name = {}
+            if not isinstance(extra_metadata_fields, (list, tuple)):
+                extra_metadata_fields = [extra_metadata_fields]
+            for ext_metadata in extra_metadata_fields:
+                ext_metadata.setdefault(
+                    'sample_type', self.configurations['sample_type'])
+                ext_metadata.setdefault('unit', self.configurations['unit'])
+                ext_metadata.setdefault(
+                    'value_attribute', ext_metadata.get(
+                        'value', self.configurations['value_attribute']))
+                ext_metadata['base_metadata'] = {
+                    'extra_metadata_captured': extra_metadata_samples,
+                    'extra_metadata_by_name': extra_metadata_by_name,
+                    'sample': request_sample
+                }
+                parent_cache_ttl = self.configurations[
+                    'extra_metadata_fields_cache_seconds']
+                cache_ttl = ext_metadata.get(
+                    'extra_metadata_fields_cache_seconds', parent_cache_ttl
+                )
+                response_cache = self.response_cache
+                extra_metadata_pollster = DynamicPollster(
+                    ext_metadata, conf=pollster_conf, cache_ttl=cache_ttl,
+                    extra_metadata_responses_cache=response_cache,
+                )
+                resources = [None]
+                if ext_metadata.get('endpoint_type'):
+                    resources = manager.discover([
+                        extra_metadata_pollster.default_discovery], {})
+                samples = extra_metadata_pollster.get_samples(
+                    manager, None, resources)
+                for sample in samples:
+                    self.fill_extra_metadata_samples(
+                        extra_metadata_by_name,
+                        extra_metadata_samples,
+                        sample)
+            return extra_metadata_samples
 
         LOG.debug("No extra metadata to be captured for pollsters [%s] and "
                   "request sample [%s].", self.definitions, request_sample)
         return {}
 
-    def _retrieve_extra_metadata(
-            self, extra_metadata_fields, manager, request_sample):
-        LOG.debug("Processing extra metadata fields [%s] for "
-                  "sample [%s].", extra_metadata_fields,
-                  request_sample)
-
-        extra_metadata_captured = {}
-        for extra_metadata in extra_metadata_fields:
-            extra_metadata_name = extra_metadata['name']
-
-            if extra_metadata_name in extra_metadata_captured.keys():
-                LOG.warning("Duplicated extra metadata name [%s]. Therefore, "
-                            "we do not process this iteration [%s].",
-                            extra_metadata_name, extra_metadata)
+    def fill_extra_metadata_samples(self, extra_metadata_by_name,
+                                    extra_metadata_samples, sample):
+        extra_metadata_samples[sample.name] = sample.volume
+        LOG.debug("Merging the sample metadata [%s] of the "
+                  "extra_metadata_field [%s], with the "
+                  "extra_metadata_samples [%s].",
+                  sample.resource_metadata,
+                  sample.name,
+                  extra_metadata_samples)
+        for key, value in sample.resource_metadata.items():
+            if value is None and key in extra_metadata_samples:
+                LOG.debug("Metadata [%s] for extra_metadata_field [%s] "
+                          "is None, skipping metadata override by None "
+                          "value", key, sample.name)
                 continue
+            extra_metadata_samples[key] = value
+        extra_metadata_by_name[sample.name] = {
+            'value': sample.volume,
+            'metadata': sample.resource_metadata
+        }
 
-            LOG.debug("Processing extra metadata [%s] for sample [%s].",
-                      extra_metadata_name, request_sample)
-
-            endpoint_type = 'endpoint:' + extra_metadata['endpoint_type']
-            if not endpoint_type.endswith(
-                    PollsterDefinitions.EXTERNAL_ENDPOINT_TYPE):
-                response = self.execute_openstack_extra_metadata_gathering(
-                    endpoint_type, extra_metadata, manager, request_sample,
-                    extra_metadata_captured)
-            else:
-                raise declarative.NonOpenStackApisDynamicPollsterException(
-                    "Not supported the use of extra metadata gathering for "
-                    "non-openstack endpoints [%s] (yet)." % extra_metadata)
-
-            extra_metadata_extractor_kwargs = {
-                'value_attribute': extra_metadata['value'],
-                'sample': request_sample}
-
-            extra_metadata_value = \
-                self.sample_extractor.retrieve_attribute_nested_value(
-                    response, **extra_metadata_extractor_kwargs)
-
-            LOG.debug("Generated extra metadata [%s] with value [%s].",
-                      extra_metadata_name, extra_metadata_value)
-            extra_metadata_captured[extra_metadata_name] = extra_metadata_value
-
-        return extra_metadata_captured
-
-    def execute_openstack_extra_metadata_gathering(self, endpoint_type,
-                                                   extra_metadata, manager,
-                                                   request_sample,
-                                                   extra_metadata_captured):
-        url_for_endpoint_type = manager.discover(
-            [endpoint_type], self.response_cache)
-
-        LOG.debug("URL [%s] found for endpoint type [%s].",
-                  url_for_endpoint_type, endpoint_type)
-
-        if url_for_endpoint_type:
-            url_for_endpoint_type = url_for_endpoint_type[0]
-
-        self.sample_gatherer.generate_url_path(
-            extra_metadata, request_sample, extra_metadata_captured)
-
-        cached_response, max_ttl_for_cache = self.response_cache.get(
-            extra_metadata['url_path'], (None, None))
-
-        extra_metadata_fields_cache_seconds = extra_metadata.get(
-            'extra_metadata_fields_cache_seconds',
-            self.configurations['extra_metadata_fields_cache_seconds'])
-
-        current_time = time.time()
-        if cached_response and max_ttl_for_cache >= current_time:
-            LOG.debug("Returning response [%s] for request [%s] as the TTL "
-                      "[max=%s, current_time=%s] has not expired yet.",
-                      cached_response, extra_metadata['url_path'],
-                      max_ttl_for_cache, current_time)
-            return cached_response
-
-        if cached_response:
-            LOG.debug("Cleaning cached response [%s] for request [%s] "
-                      "as the TTL [max=%s, current_time=%s] has expired.",
-                      cached_response, extra_metadata['url_path'],
-                      max_ttl_for_cache, current_time)
-
-        response = self.sample_gatherer.execute_request_for_definitions(
-            extra_metadata, **{'manager': manager,
-                               'keystone_client': manager._keystone,
-                               'resource': url_for_endpoint_type,
-                               'execute_id_overrides': False})
-
-        max_ttl_for_cache = time.time() + extra_metadata_fields_cache_seconds
-
-        cache_tuple = (response, max_ttl_for_cache)
-        self.response_cache[extra_metadata['url_path']] = cache_tuple
-        return response
+        LOG.debug("extra_metadata_samples after merging: [%s].",
+                  extra_metadata_samples)
 
 
 class MultiMetricPollsterDefinitions(PollsterDefinitions):
@@ -739,6 +702,42 @@ class PollsterSampleGatherer(object):
             url_path=definitions.configurations['url_path']
         )
 
+    def get_cache_key(self, definitions, **kwargs):
+        return self.get_request_linked_samples_url(kwargs, definitions)
+
+    def get_cached_response(self, definitions, **kwargs):
+        if self.definitions.cache_ttl == 0:
+            return
+        cache_key = self.get_cache_key(definitions, **kwargs)
+        response_cache = self.definitions.response_cache
+        cached_response, max_ttl_for_cache = response_cache.get(
+            cache_key, (None, None))
+
+        current_time = time.time()
+        if cached_response and max_ttl_for_cache >= current_time:
+            LOG.debug("Returning response [%s] for request [%s] as the TTL "
+                      "[max=%s, current_time=%s] has not expired yet.",
+                      cached_response, definitions,
+                      max_ttl_for_cache, current_time)
+            return cached_response
+
+        if cached_response and max_ttl_for_cache < current_time:
+            LOG.debug("Cleaning cached response [%s] for request [%s] "
+                      "as the TTL [max=%s, current_time=%s] has expired.",
+                      cached_response, definitions,
+                      max_ttl_for_cache, current_time)
+            response_cache.pop(cache_key, None)
+
+    def store_cached_response(self, definitions, resp, **kwargs):
+        if self.definitions.cache_ttl == 0:
+            return
+        cache_key = self.get_cache_key(definitions, **kwargs)
+        extra_metadata_fields_cache_seconds = self.definitions.cache_ttl
+        max_ttl_for_cache = time.time() + extra_metadata_fields_cache_seconds
+
+        cache_tuple = (resp, max_ttl_for_cache)
+        self.definitions.response_cache[cache_key] = cache_tuple
+
     @property
     def default_discovery(self):
         return 'endpoint:' + self.definitions.configurations['endpoint_type']
@@ -748,10 +747,14 @@ class PollsterSampleGatherer(object):
             self.definitions.configurations, **kwargs)
 
     def execute_request_for_definitions(self, definitions, **kwargs):
-        resp, url = self._internal_execute_request_get_samples(
-            definitions=definitions, **kwargs)
+        if response_dict := self.get_cached_response(definitions, **kwargs):
+            url = 'cached'
+        else:
+            resp, url = self._internal_execute_request_get_samples(
+                definitions=definitions, **kwargs)
+            response_dict = self.response_handler_chain.handle(resp.text)
+            self.store_cached_response(definitions, response_dict, **kwargs)
 
-        response_dict = self.response_handler_chain.handle(resp.text)
         entry_size = len(response_dict)
         LOG.debug("Entries [%s] in the DICT for request [%s] "
                   "for dynamic pollster [%s].",
@@ -790,21 +793,6 @@ class PollsterSampleGatherer(object):
                 self.generate_new_attributes_in_sample(
                     request_sample, resource_id_attribute, 'id')
 
-    def generate_url_path(self, extra_metadata, sample,
-                          extra_metadata_captured):
-        if not extra_metadata.get('url_path_original'):
-            extra_metadata[
-                'url_path_original'] = extra_metadata['url_path']
-
-        extra_metadata['url_path'] = eval(
-            extra_metadata['url_path_original'])
-
-        LOG.debug("URL [%s] generated for pattern [%s] for sample [%s] and "
-                  "extra metadata captured [%s].",
-                  extra_metadata['url_path'],
-                  extra_metadata['url_path_original'], sample,
-                  extra_metadata_captured)
-
     def generate_new_attributes_in_sample(
             self, sample, attribute_key, new_attribute_key):
 
@@ -881,6 +869,15 @@ class PollsterSampleGatherer(object):
 
     def get_request_url(self, kwargs, url_path):
         endpoint = kwargs['resource']
+        params = copy.deepcopy(
+            self.definitions.configurations.get(
+                'base_metadata', {}))
+        try:
+            url_path = eval(url_path, params)
+        except Exception:
+            LOG.debug("Cannot eval path [%s] with params [%s],"
+                      " using [%s] instead.",
+                      url_path, params, url_path)
         return urlparse.urljoin(endpoint, url_path)
 
     def retrieve_entries_from_response(self, response_json, definitions):
@@ -919,6 +916,57 @@ class NonOpenStackApisPollsterDefinition(PollsterDefinitions):
         return configurations.get('module')
 
 
+class HostCommandPollsterDefinition(PollsterDefinitions):
+
+    extra_definitions = [
+        PollsterDefinition(name='endpoint_type', required=False),
+        PollsterDefinition(name='url_path', required=False),
+        PollsterDefinition(name='host_command', required=True)]
+
+    def __init__(self, configurations):
+        super(HostCommandPollsterDefinition, self).__init__(
+            configurations)
+        self.sample_gatherer = HostCommandSamplesGatherer(self)
+
+    @staticmethod
+    def is_field_applicable_to_definition(configurations):
+        return configurations.get('host_command')
+
+
+class HostCommandSamplesGatherer(PollsterSampleGatherer):
+
+    class Response(object):
+        def __init__(self, text):
+            self.text = text
+
+    def get_cache_key(self, definitions, **kwargs):
+        return self.get_command(definitions)
+
+    def _internal_execute_request_get_samples(self, definitions, **kwargs):
+        command = self.get_command(definitions, **kwargs)
+        LOG.debug('Running Host command: [%s]', command)
+        result = subprocess.getoutput(command)
+        LOG.debug('Host command [%s] result: [%s]', command, result)
+        return self.Response(result), command
+
+    def get_command(self, definitions, next_sample_url=None, **kwargs):
+        command = next_sample_url or definitions['host_command']
+        params = copy.deepcopy(
+            self.definitions.configurations.get(
+                'base_metadata', {}))
+        try:
+            command = eval(command, params)
+        except Exception:
+            LOG.debug("Cannot eval command [%s] with params [%s],"
+                      " using [%s] instead.",
+                      command, params, command)
+        return command
+
+    @property
+    def default_discovery(self):
+        return 'local_node'
+
+
 class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
 
     @property
@@ -996,8 +1044,10 @@ class DynamicPollster(plugin_base.PollsterBase):
     # Mandatory name field
     name = ""
 
-    def __init__(self, pollster_definitions={}, conf=None,
-                 supported_definitions=[NonOpenStackApisPollsterDefinition,
+    def __init__(self, pollster_definitions={}, conf=None, cache_ttl=0,
+                 extra_metadata_responses_cache=None,
+                 supported_definitions=[HostCommandPollsterDefinition,
+                                        NonOpenStackApisPollsterDefinition,
                                         MultiMetricPollsterDefinitions,
                                         SingleMetricPollsterDefinitions]):
         super(DynamicPollster, self).__init__(conf)
@@ -1007,6 +1057,10 @@ class DynamicPollster(plugin_base.PollsterBase):
 
         self.definitions = PollsterDefinitionBuilder(
             self.supported_definitions).build_definitions(pollster_definitions)
+        self.definitions.cache_ttl = cache_ttl
+        self.definitions.response_cache = extra_metadata_responses_cache
+        if extra_metadata_responses_cache is None:
+            self.definitions.response_cache = {}
         self.pollster_definitions = self.definitions.configurations
         if 'metadata_fields' in self.pollster_definitions:
             LOG.debug("Metadata fields configured to [%s].",
@@ -1040,9 +1094,12 @@ class DynamicPollster(plugin_base.PollsterBase):
         for r in resources:
             LOG.debug("Executing get sample for resource [%s].", r)
             samples = self.load_samples(r, manager)
+            if not isinstance(samples, (list, tuple)):
+                samples = [samples]
             for pollster_sample in samples:
-                kwargs = {'manager': manager, 'resource': r}
-                sample = self.extract_sample(pollster_sample, **kwargs)
+                sample = self.extract_sample(
+                    pollster_sample, manager=manager,
+                    resource=r, conf=self.conf)
                 if isinstance(sample, SkippedSample):
                     continue
                 yield from sample
diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
index dfc0c576..653f2df2 100644
--- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
@@ -389,6 +389,424 @@ class TestDynamicPollster(base.BaseTestCase):
         self.assertEqual(4, len(samples))
 
     @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_extra_metadata_fields_cache_disabled(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        extra_metadata_fields = {
+            'extra_metadata_fields_cache_seconds': 0,
+            'name': "project_name",
+            'endpoint_type': "identity",
+            'url_path': "'/v3/projects/' + str(sample['project_id'])",
+            'value': "name",
+        }
+        definitions['value_attribute'] = 'project_id'
+        definitions['extra_metadata_fields'] = extra_metadata_fields
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '''
+                    {"projects": [
+                        {"project_id": 9999, "name": "project1"},
+                        {"project_id": 8888, "name": "project2"},
+                        {"project_id": 7777, "name": "project3"},
+                        {"project_id": 9999, "name": "project1"},
+                        {"project_id": 8888, "name": "project2"},
+                        {"project_id": 7777, "name": "project3"},
+                        {"project_id": 9999, "name": "project1"},
+                        {"project_id": 8888, "name": "project2"},
+                        {"project_id": 7777, "name": "project3"}]
+                    }
+                '''
+
+        return_value9999 = self.FakeResponse()
+        return_value9999.status_code = requests.codes.ok
+        return_value9999._text = '''
+                    {"project":
+                        {"project_id": 9999, "name": "project1"}
+                    }
+                '''
+
+        return_value8888 = self.FakeResponse()
+        return_value8888.status_code = requests.codes.ok
+        return_value8888._text = '''
+                    {"project":
+                        {"project_id": 8888, "name": "project2"}
+                    }
+                '''
+
+        return_value7777 = self.FakeResponse()
+        return_value7777.status_code = requests.codes.ok
+        return_value7777._text = '''
+                    {"project":
+                        {"project_id": 7777, "name": "project3"}
+                    }
+                '''
+
+        def get(url, *args, **kwargs):
+            if '9999' in url:
+                return return_value9999
+            if '8888' in url:
+                return return_value8888
+            if '7777' in url:
+                return return_value7777
+            return return_value
+
+        client_mock.session.get.side_effect = get
+        manager = mock.Mock
+        manager._keystone = client_mock
+
+        def discover(*args, **kwargs):
+            return ["https://endpoint.server.name/"]
+
+        manager.discover = discover
+        samples = pollster.get_samples(
+            manager=manager, cache=None,
+            resources=["https://endpoint.server.name/"])
+
+        samples = list(samples)
+
+        n_calls = client_mock.session.get.call_count
+        self.assertEqual(9, len(samples))
+        self.assertEqual(10, n_calls)
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_extra_metadata_fields_cache_enabled(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        extra_metadata_fields = {
+            'extra_metadata_fields_cache_seconds': 3600,
+            'name': "project_name",
+            'endpoint_type': "identity",
+            'url_path': "'/v3/projects/' + str(sample['project_id'])",
+            'value': "name",
+        }
+        definitions['value_attribute'] = 'project_id'
+        definitions['extra_metadata_fields'] = extra_metadata_fields
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '''
+                    {"projects": [
+                        {"project_id": 9999, "name": "project1"},
+                        {"project_id": 8888, "name": "project2"},
+                        {"project_id": 7777, "name": "project3"},
+                        {"project_id": 9999, "name": "project4"},
+                        {"project_id": 8888, "name": "project5"},
+                        {"project_id": 7777, "name": "project6"},
+                        {"project_id": 9999, "name": "project7"},
+                        {"project_id": 8888, "name": "project8"},
+                        {"project_id": 7777, "name": "project9"}]
+                    }
+                '''
+
+        return_value9999 = self.FakeResponse()
+        return_value9999.status_code = requests.codes.ok
+        return_value9999._text = '''
+                    {"project":
+                        {"project_id": 9999, "name": "project1"}
+                    }
+                '''
+
+        return_value8888 = self.FakeResponse()
+        return_value8888.status_code = requests.codes.ok
+        return_value8888._text = '''
+                    {"project":
+                        {"project_id": 8888, "name": "project2"}
+                    }
+                '''
+
+        return_value7777 = self.FakeResponse()
+        return_value7777.status_code = requests.codes.ok
+        return_value7777._text = '''
+                    {"project":
+                        {"project_id": 7777, "name": "project3"}
+                    }
+                '''
+
+        def get(url, *args, **kwargs):
+            if '9999' in url:
+                return return_value9999
+            if '8888' in url:
+                return return_value8888
+            if '7777' in url:
+                return return_value7777
+            return return_value
+
+        client_mock.session.get.side_effect = get
+        manager = mock.Mock
+        manager._keystone = client_mock
+
+        def discover(*args, **kwargs):
+            return ["https://endpoint.server.name/"]
+
+        manager.discover = discover
+        samples = pollster.get_samples(
+            manager=manager, cache=None,
+            resources=["https://endpoint.server.name/"])
+
+        samples = list(samples)
+
+        n_calls = client_mock.session.get.call_count
+        self.assertEqual(9, len(samples))
+        self.assertEqual(4, n_calls)
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_extra_metadata_fields(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        extra_metadata_fields = [{
+            'name': "project_name",
+            'endpoint_type': "identity",
+            'url_path': "'/v3/projects/' + str(sample['project_id'])",
+            'value': "name",
+            'metadata_fields': ['meta']
+        }, {
+            'name': "project_alias",
+            'endpoint_type': "identity",
+            'url_path': "'/v3/projects/' + "
+                        "str(extra_metadata_captured['project_name'])",
+            'value': "name",
+            'metadata_fields': ['meta']
+        }, {
+            'name': "project_meta",
+            'endpoint_type': "identity",
+            'url_path': "'/v3/projects/' + "
+                        "str(extra_metadata_by_name['project_name']"
+                        "['metadata']['meta'])",
+            'value': "project_id",
+            'metadata_fields': ['meta']
+        }]
+        definitions['value_attribute'] = 'project_id'
+        definitions['extra_metadata_fields'] = extra_metadata_fields
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '''
+                    {"projects": [
+                        {"project_id": 9999, "name": "project1"},
+                        {"project_id": 8888, "name": "project2"},
+                        {"project_id": 7777, "name": "project3"}]
+                    }
+                '''
+
+        return_value9999 = self.FakeResponse()
+        return_value9999.status_code = requests.codes.ok
+        return_value9999._text = '''
+                    {"project":
+                        {"project_id": 9999, "name": "project1",
+                        "meta": "m1"}
+                    }
+                '''
+
+        return_value8888 = self.FakeResponse()
+        return_value8888.status_code = requests.codes.ok
+        return_value8888._text = '''
+                    {"project":
+                        {"project_id": 8888, "name": "project2",
+                        "meta": "m2"}
+                    }
+                '''
+
+        return_value7777 = self.FakeResponse()
+        return_value7777.status_code = requests.codes.ok
+        return_value7777._text = '''
+                    {"project":
+                        {"project_id": 7777, "name": "project3",
+                        "meta": "m3"}
+                    }
+                '''
+
+        return_valueP1 = self.FakeResponse()
+        return_valueP1.status_code = requests.codes.ok
+        return_valueP1._text = '''
+                    {"project":
+                        {"project_id": 7777, "name": "p1",
+                        "meta": null}
+                    }
+                '''
+
+        return_valueP2 = self.FakeResponse()
+        return_valueP2.status_code = requests.codes.ok
+        return_valueP2._text = '''
+                    {"project":
+                        {"project_id": 7777, "name": "p2",
+                        "meta": null}
+                    }
+                '''
+
+        return_valueP3 = self.FakeResponse()
+        return_valueP3.status_code = requests.codes.ok
+        return_valueP3._text = '''
+                    {"project":
+                        {"project_id": 7777, "name": "p3",
+                        "meta": null}
+                    }
+                '''
+
+        return_valueM1 = self.FakeResponse()
+        return_valueM1.status_code = requests.codes.ok
+        return_valueM1._text = '''
+                    {"project":
+                        {"project_id": "META1", "name": "p3",
+                        "meta": null}
+                    }
+                '''
+
+        return_valueM2 = self.FakeResponse()
+        return_valueM2.status_code = requests.codes.ok
+        return_valueM2._text = '''
+                    {"project":
+                        {"project_id": "META2", "name": "p3",
+                        "meta": null}
+                    }
+                '''
+
+        return_valueM3 = self.FakeResponse()
+        return_valueM3.status_code = requests.codes.ok
+        return_valueM3._text = '''
+                    {"project":
+                        {"project_id": "META3", "name": "p3",
+                        "meta": null}
+                    }
+                '''
+
+        def get(url, *args, **kwargs):
+            if '9999' in url:
+                return return_value9999
+            if '8888' in url:
+                return return_value8888
+            if '7777' in url:
+                return return_value7777
+            if 'project1' in url:
+                return return_valueP1
+            if 'project2' in url:
+                return return_valueP2
+            if 'project3' in url:
+                return return_valueP3
+            if 'm1' in url:
+                return return_valueM1
+            if 'm2' in url:
+                return return_valueM2
+            if 'm3' in url:
+                return return_valueM3
+
+            return return_value
+
+        client_mock.session.get = get
+        manager = mock.Mock
+        manager._keystone = client_mock
+
+        def discover(*args, **kwargs):
+            return ["https://endpoint.server.name/"]
+
+        manager.discover = discover
+        samples = pollster.get_samples(
+            manager=manager, cache=None,
+            resources=["https://endpoint.server.name/"])
+
+        samples = list(samples)
+        self.assertEqual(3, len(samples))
+
+        self.assertEqual(samples[0].volume, 9999)
+        self.assertEqual(samples[1].volume, 8888)
+        self.assertEqual(samples[2].volume, 7777)
+
+        self.assertEqual(samples[0].resource_metadata,
+                         {'project_name': 'project1',
+                          'project_alias': 'p1',
+                          'meta': 'm1',
+                          'project_meta': 'META1'})
+        self.assertEqual(samples[1].resource_metadata,
+                         {'project_name': 'project2',
+                          'project_alias': 'p2',
+                          'meta': 'm2',
+                          'project_meta': 'META2'})
+        self.assertEqual(samples[2].resource_metadata,
+                         {'project_name': 'project3',
+                          'project_alias': 'p3',
+                          'meta': 'm3',
+                          'project_meta': 'META3'})
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_extra_metadata_fields_different_requests(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+
+        command = ''' \'\'\'echo '{"project":
+        {"project_id": \'\'\'+ str(sample['project_id'])
+         +\'\'\' , "name": "project1"}}' \'\'\' '''.replace('\n', '')
+
+        command2 = ''' \'\'\'echo '{"project":
+        {"project_id": \'\'\'+ str(sample['project_id'])
+         +\'\'\' , "name": "project2"}}' \'\'\' '''.replace('\n', '')
+
+        extra_metadata_fields_embedded = {
+            'name': "project_name2",
+            'host_command': command2,
+            'value': "name",
+        }
+
+        extra_metadata_fields = {
+            'name': "project_id2",
+            'host_command': command,
+            'value': "project_id",
+            'extra_metadata_fields': extra_metadata_fields_embedded
+        }
+
+        definitions['value_attribute'] = 'project_id'
+        definitions['extra_metadata_fields'] = extra_metadata_fields
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '''
+                    {"projects": [
+                        {"project_id": 9999, "name": "project1"},
+                        {"project_id": 8888, "name": "project2"},
+                        {"project_id": 7777, "name": "project3"}]
+                    }
+                '''
+
+        def get(url, *args, **kwargs):
+            return return_value
+
+        client_mock.session.get = get
+        manager = mock.Mock
+        manager._keystone = client_mock
+
+        def discover(*args, **kwargs):
+            return ["https://endpoint.server.name/"]
+
+        manager.discover = discover
+        samples = pollster.get_samples(
+            manager=manager, cache=None,
+            resources=["https://endpoint.server.name/"])
+
+        samples = list(samples)
+        self.assertEqual(3, len(samples))
+
+        self.assertEqual(samples[0].volume, 9999)
+        self.assertEqual(samples[1].volume, 8888)
+        self.assertEqual(samples[2].volume, 7777)
+
+        self.assertEqual(samples[0].resource_metadata,
+                         {'project_id2': 9999,
+                          'project_name2': 'project2'})
+        self.assertEqual(samples[1].resource_metadata,
+                         {'project_id2': 8888,
+                          'project_name2': 'project2'})
+        self.assertEqual(samples[2].resource_metadata,
+                         {'project_id2': 7777,
+                          'project_name2': 'project2'})
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
     def test_execute_request_xml_json_response_handler_invalid_response(
             self, client_mock):
         definitions = copy.deepcopy(
@@ -410,8 +828,8 @@ class TestDynamicPollster(base.BaseTestCase):
                 keystone_client=client_mock,
                 resource="https://endpoint.server.name/")
 
-            xml_handling_error = logs.output[2]
-            json_handling_error = logs.output[3]
+            xml_handling_error = logs.output[3]
+            json_handling_error = logs.output[4]
 
             self.assertIn(
                 'DEBUG:ceilometer.polling.dynamic_pollster:'
@@ -479,6 +897,57 @@ class TestDynamicPollster(base.BaseTestCase):
                                       resource="https://endpoint.server.name/")
         self.assertEqual("Mock HTTP error.", str(exception))
 
+    def test_execute_host_command_paged_responses(self):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['host_command'] = '''
+        echo '{"server": [{"status": "ACTIVE"}], "next": ""}'
+        '''
+        str_json = "'{\\\"server\\\": [{\\\"status\\\": \\\"INACTIVE\\\"}]}'"
+        definitions['next_sample_url_attribute'] = \
+            "next|\"echo \"+value+\"" + str_json + '"'
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+        samples = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples()
+        resp_json = [{'status': 'ACTIVE'}, {'status': 'INACTIVE'}]
+        self.assertEqual(resp_json, samples)
+
+    def test_execute_host_command_response_handler(self):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = ['xml', 'json']
+        definitions['host_command'] = 'echo "<a><y>xml\n</y><s>xml</s></a>"'
+        entry = 'a'
+        definitions['response_entries_key'] = entry
+        definitions.pop('url_path')
+        definitions.pop('endpoint_type')
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        samples_xml = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples()
+
+        definitions['host_command'] = 'echo \'{"a": {"y":"json",' \
+                                      '\n"s":"json"}}\''
+        samples_json = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples()
+
+        resp_xml = {'a': {'y': 'xml', 's': 'xml'}}
+        resp_json = {'a': {'y': 'json', 's': 'json'}}
+        self.assertEqual(resp_xml[entry], samples_xml)
+        self.assertEqual(resp_json[entry], samples_json)
+
+    def test_execute_host_command_invalid_command(self):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['host_command'] = 'invalid-command'
+        definitions.pop('url_path')
+        definitions.pop('endpoint_type')
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        self.assertRaises(
+            declarative.InvalidResponseTypeException,
+            pollster.definitions.sample_gatherer.execute_request_get_samples)
+
     def test_generate_new_metadata_fields_no_metadata_mapping(self):
         metadata = {'name': 'someName',
                     'value': 1}
@@ -1105,7 +1574,7 @@ class TestDynamicPollster(base.BaseTestCase):
 
         sample = pollster.definitions.sample_extractor.generate_sample(
             pollster_sample, pollster.definitions.configurations,
-            manager=mock.Mock())
+            manager=mock.Mock(), conf={})
 
         self.assertEqual(1, sample.volume)
         self.assertEqual(2, len(sample.resource_metadata))
@@ -1127,7 +1596,7 @@ class TestDynamicPollster(base.BaseTestCase):
 
         sample = pollster.definitions.sample_extractor.generate_sample(
             pollster_sample, pollster.definitions.configurations,
-            manager=mock.Mock())
+            manager=mock.Mock(), conf={})
 
         self.assertEqual(1, sample.volume)
         self.assertEqual(3, len(sample.resource_metadata))
@@ -1150,7 +1619,7 @@ class TestDynamicPollster(base.BaseTestCase):
 
         sample = pollster.definitions.sample_extractor.generate_sample(
             pollster_sample, pollster.definitions.configurations,
-            manager=mock.Mock())
+            manager=mock.Mock(), conf={})
 
         self.assertEqual(1, sample.volume)
         self.assertEqual(3, len(sample.resource_metadata))