diff options
author | Pedro Henrique <phpm13@gmail.com> | 2022-08-03 10:09:00 -0300 |
---|---|---|
committer | Pedro Henrique <phpm13@gmail.com> | 2022-09-13 15:04:58 -0300 |
commit | cb448a1dbc48ddd28d9fd0f6b6e44deafd636f52 (patch) | |
tree | 41a0b1a7bab96381b8093661918a049aa0476a28 /ceilometer/tests | |
parent | 225f1cd7765ddb7b725c538944947ada8c52e73f (diff) | |
download | ceilometer-cb448a1dbc48ddd28d9fd0f6b6e44deafd636f52.tar.gz |
Add support to host command dynamic pollster definitions
Problem description
===================
Today we have some hardcoded pollsters that are gathering
data from running virtual machines through libvirt or
different programs running in the compute nodes. However,
the Dynamic pollster definition does not support this kind of
operations to gather data, it only supports HTTP Rest
requests to collect data. Therefore, it is not possible to
use the dynamic pollster definition to create a YML based
pollster that runs and collects data from Libvirt in the
compute nodes.
Proposal
========
To allow host commands/scripts in the Dynamic pollsters,
we propose to add a new pollster definition using the
`os.subprocess` lib to run host commands to collect
Host/VMs data and store them in the configured backend.
This will provide more flexibility and make the
Dynamic pollsters able to be used in Ceilometer compute
instances as well.
Change-Id: I50b8dc341ce457780416b41d138e35f5a0d083b6
Depends-On: https://review.opendev.org/c/openstack/ceilometer/+/850253
Diffstat (limited to 'ceilometer/tests')
-rw-r--r-- | ceilometer/tests/unit/polling/test_dynamic_pollster.py | 479 |
1 files changed, 474 insertions, 5 deletions
diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index dfc0c576..653f2df2 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -389,6 +389,424 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(4, len(samples)) @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_extra_metadata_fields_cache_disabled( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + extra_metadata_fields = { + 'extra_metadata_fields_cache_seconds': 0, + 'name': "project_name", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + str(sample['project_id'])", + 'value': "name", + } + definitions['value_attribute'] = 'project_id' + definitions['extra_metadata_fields'] = extra_metadata_fields + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = ''' + {"projects": [ + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}, + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}, + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}] + } + ''' + + return_value9999 = self.FakeResponse() + return_value9999.status_code = requests.codes.ok + return_value9999._text = ''' + {"project": + {"project_id": 9999, "name": "project1"} + } + ''' + + return_value8888 = self.FakeResponse() + return_value8888.status_code = requests.codes.ok + return_value8888._text = ''' + {"project": + {"project_id": 8888, "name": "project2"} + } + ''' + + return_value7777 = self.FakeResponse() + return_value7777.status_code = requests.codes.ok + return_value7777._text = ''' + {"project": + {"project_id": 7777, "name": "project3"} + } + ''' + + def get(url, *args, **kwargs): + if '9999' in url: + return return_value9999 + if '8888' in url: + return return_value8888 + if '7777' in url: + return return_value7777 + return return_value + + client_mock.session.get.side_effect = get + manager = mock.Mock + manager._keystone = client_mock + + def discover(*args, **kwargs): + return ["https://endpoint.server.name/"] + + manager.discover = discover + samples = pollster.get_samples( + manager=manager, cache=None, + resources=["https://endpoint.server.name/"]) + + samples = list(samples) + + n_calls = client_mock.session.get.call_count + self.assertEqual(9, len(samples)) + self.assertEqual(10, n_calls) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_extra_metadata_fields_cache_enabled( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + extra_metadata_fields = { + 'extra_metadata_fields_cache_seconds': 3600, + 'name': "project_name", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + str(sample['project_id'])", + 'value': "name", + } + definitions['value_attribute'] = 'project_id' + definitions['extra_metadata_fields'] = extra_metadata_fields + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = ''' + {"projects": [ + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}, + {"project_id": 9999, "name": "project4"}, + {"project_id": 8888, "name": "project5"}, + {"project_id": 7777, "name": "project6"}, + {"project_id": 9999, "name": "project7"}, + {"project_id": 8888, "name": "project8"}, + {"project_id": 7777, "name": "project9"}] + } + ''' + + return_value9999 = self.FakeResponse() + return_value9999.status_code = requests.codes.ok + return_value9999._text = ''' + {"project": + {"project_id": 9999, "name": "project1"} + } + ''' + + return_value8888 = self.FakeResponse() + return_value8888.status_code = requests.codes.ok + return_value8888._text = ''' + {"project": + {"project_id": 8888, "name": "project2"} + } + ''' + + return_value7777 = self.FakeResponse() + return_value7777.status_code = requests.codes.ok + return_value7777._text = ''' + {"project": + {"project_id": 7777, "name": "project3"} + } + ''' + + def get(url, *args, **kwargs): + if '9999' in url: + return return_value9999 + if '8888' in url: + return return_value8888 + if '7777' in url: + return return_value7777 + return return_value + + client_mock.session.get.side_effect = get + manager = mock.Mock + manager._keystone = client_mock + + def discover(*args, **kwargs): + return ["https://endpoint.server.name/"] + + manager.discover = discover + samples = pollster.get_samples( + manager=manager, cache=None, + resources=["https://endpoint.server.name/"]) + + samples = list(samples) + + n_calls = client_mock.session.get.call_count + self.assertEqual(9, len(samples)) + self.assertEqual(4, n_calls) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_extra_metadata_fields( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + extra_metadata_fields = [{ + 'name': "project_name", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + str(sample['project_id'])", + 'value': "name", + 'metadata_fields': ['meta'] + }, { + 'name': "project_alias", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + " + "str(extra_metadata_captured['project_name'])", + 'value': "name", + 'metadata_fields': ['meta'] + }, { + 'name': "project_meta", + 'endpoint_type': "identity", + 'url_path': "'/v3/projects/' + " + "str(extra_metadata_by_name['project_name']" + "['metadata']['meta'])", + 'value': "project_id", + 'metadata_fields': ['meta'] + }] + definitions['value_attribute'] = 'project_id' + definitions['extra_metadata_fields'] = extra_metadata_fields + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = ''' + {"projects": [ + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}] + } + ''' + + return_value9999 = self.FakeResponse() + return_value9999.status_code = requests.codes.ok + return_value9999._text = ''' + {"project": + {"project_id": 9999, "name": "project1", + "meta": "m1"} + } + ''' + + return_value8888 = self.FakeResponse() + return_value8888.status_code = requests.codes.ok + return_value8888._text = ''' + {"project": + {"project_id": 8888, "name": "project2", + "meta": "m2"} + } + ''' + + return_value7777 = self.FakeResponse() + return_value7777.status_code = requests.codes.ok + return_value7777._text = ''' + {"project": + {"project_id": 7777, "name": "project3", + "meta": "m3"} + } + ''' + + return_valueP1 = self.FakeResponse() + return_valueP1.status_code = requests.codes.ok + return_valueP1._text = ''' + {"project": + {"project_id": 7777, "name": "p1", + "meta": null} + } + ''' + + return_valueP2 = self.FakeResponse() + return_valueP2.status_code = requests.codes.ok + return_valueP2._text = ''' + {"project": + {"project_id": 7777, "name": "p2", + "meta": null} + } + ''' + + return_valueP3 = self.FakeResponse() + return_valueP3.status_code = requests.codes.ok + return_valueP3._text = ''' + {"project": + {"project_id": 7777, "name": "p3", + "meta": null} + } + ''' + + return_valueM1 = self.FakeResponse() + return_valueM1.status_code = requests.codes.ok + return_valueM1._text = ''' + {"project": + {"project_id": "META1", "name": "p3", + "meta": null} + } + ''' + + return_valueM2 = self.FakeResponse() + return_valueM2.status_code = requests.codes.ok + return_valueM2._text = ''' + {"project": + {"project_id": "META2", "name": "p3", + "meta": null} + } + ''' + + return_valueM3 = self.FakeResponse() + return_valueM3.status_code = requests.codes.ok + return_valueM3._text = ''' + {"project": + {"project_id": "META3", "name": "p3", + "meta": null} + } + ''' + + def get(url, *args, **kwargs): + if '9999' in url: + return return_value9999 + if '8888' in url: + return return_value8888 + if '7777' in url: + return return_value7777 + if 'project1' in url: + return return_valueP1 + if 'project2' in url: + return return_valueP2 + if 'project3' in url: + return return_valueP3 + if 'm1' in url: + return return_valueM1 + if 'm2' in url: + return return_valueM2 + if 'm3' in url: + return return_valueM3 + + return return_value + + client_mock.session.get = get + manager = mock.Mock + manager._keystone = client_mock + + def discover(*args, **kwargs): + return ["https://endpoint.server.name/"] + + manager.discover = discover + samples = pollster.get_samples( + manager=manager, cache=None, + resources=["https://endpoint.server.name/"]) + + samples = list(samples) + self.assertEqual(3, len(samples)) + + self.assertEqual(samples[0].volume, 9999) + self.assertEqual(samples[1].volume, 8888) + self.assertEqual(samples[2].volume, 7777) + + self.assertEqual(samples[0].resource_metadata, + {'project_name': 'project1', + 'project_alias': 'p1', + 'meta': 'm1', + 'project_meta': 'META1'}) + self.assertEqual(samples[1].resource_metadata, + {'project_name': 'project2', + 'project_alias': 'p2', + 'meta': 'm2', + 'project_meta': 'META2'}) + self.assertEqual(samples[2].resource_metadata, + {'project_name': 'project3', + 'project_alias': 'p3', + 'meta': 'm3', + 'project_meta': 'META3'}) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_extra_metadata_fields_different_requests( + self, client_mock): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + + command = ''' \'\'\'echo '{"project": + {"project_id": \'\'\'+ str(sample['project_id']) + +\'\'\' , "name": "project1"}}' \'\'\' '''.replace('\n', '') + + command2 = ''' \'\'\'echo '{"project": + {"project_id": \'\'\'+ str(sample['project_id']) + +\'\'\' , "name": "project2"}}' \'\'\' '''.replace('\n', '') + + extra_metadata_fields_embedded = { + 'name': "project_name2", + 'host_command': command2, + 'value': "name", + } + + extra_metadata_fields = { + 'name': "project_id2", + 'host_command': command, + 'value': "project_id", + 'extra_metadata_fields': extra_metadata_fields_embedded + } + + definitions['value_attribute'] = 'project_id' + definitions['extra_metadata_fields'] = extra_metadata_fields + pollster = dynamic_pollster.DynamicPollster(definitions) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value._text = ''' + {"projects": [ + {"project_id": 9999, "name": "project1"}, + {"project_id": 8888, "name": "project2"}, + {"project_id": 7777, "name": "project3"}] + } + ''' + + def get(url, *args, **kwargs): + return return_value + + client_mock.session.get = get + manager = mock.Mock + manager._keystone = client_mock + + def discover(*args, **kwargs): + return ["https://endpoint.server.name/"] + + manager.discover = discover + samples = pollster.get_samples( + manager=manager, cache=None, + resources=["https://endpoint.server.name/"]) + + samples = list(samples) + self.assertEqual(3, len(samples)) + + self.assertEqual(samples[0].volume, 9999) + self.assertEqual(samples[1].volume, 8888) + self.assertEqual(samples[2].volume, 7777) + + self.assertEqual(samples[0].resource_metadata, + {'project_id2': 9999, + 'project_name2': 'project2'}) + self.assertEqual(samples[1].resource_metadata, + {'project_id2': 8888, + 'project_name2': 'project2'}) + self.assertEqual(samples[2].resource_metadata, + {'project_id2': 7777, + 'project_name2': 'project2'}) + + @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_xml_json_response_handler_invalid_response( self, client_mock): definitions = copy.deepcopy( @@ -410,8 +828,8 @@ class TestDynamicPollster(base.BaseTestCase): keystone_client=client_mock, resource="https://endpoint.server.name/") - xml_handling_error = logs.output[2] - json_handling_error = logs.output[3] + xml_handling_error = logs.output[3] + json_handling_error = logs.output[4] self.assertIn( 'DEBUG:ceilometer.polling.dynamic_pollster:' @@ -479,6 +897,57 @@ class TestDynamicPollster(base.BaseTestCase): resource="https://endpoint.server.name/") self.assertEqual("Mock HTTP error.", str(exception)) + def test_execute_host_command_paged_responses(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['host_command'] = ''' + echo '{"server": [{"status": "ACTIVE"}], "next": ""}' + ''' + str_json = "'{\\\"server\\\": [{\\\"status\\\": \\\"INACTIVE\\\"}]}'" + definitions['next_sample_url_attribute'] = \ + "next|\"echo \"+value+\"" + str_json + '"' + pollster = dynamic_pollster.DynamicPollster(definitions) + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples() + resp_json = [{'status': 'ACTIVE'}, {'status': 'INACTIVE'}] + self.assertEqual(resp_json, samples) + + def test_execute_host_command_response_handler(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['response_handlers'] = ['xml', 'json'] + definitions['host_command'] = 'echo "<a><y>xml\n</y><s>xml</s></a>"' + entry = 'a' + definitions['response_entries_key'] = entry + definitions.pop('url_path') + definitions.pop('endpoint_type') + pollster = dynamic_pollster.DynamicPollster(definitions) + + samples_xml = pollster.definitions.sample_gatherer. \ + execute_request_get_samples() + + definitions['host_command'] = 'echo \'{"a": {"y":"json",' \ + '\n"s":"json"}}\'' + samples_json = pollster.definitions.sample_gatherer. \ + execute_request_get_samples() + + resp_xml = {'a': {'y': 'xml', 's': 'xml'}} + resp_json = {'a': {'y': 'json', 's': 'json'}} + self.assertEqual(resp_xml[entry], samples_xml) + self.assertEqual(resp_json[entry], samples_json) + + def test_execute_host_command_invalid_command(self): + definitions = copy.deepcopy( + self.pollster_definition_only_required_fields) + definitions['host_command'] = 'invalid-command' + definitions.pop('url_path') + definitions.pop('endpoint_type') + pollster = dynamic_pollster.DynamicPollster(definitions) + + self.assertRaises( + declarative.InvalidResponseTypeException, + pollster.definitions.sample_gatherer.execute_request_get_samples) + def test_generate_new_metadata_fields_no_metadata_mapping(self): metadata = {'name': 'someName', 'value': 1} @@ -1105,7 +1574,7 @@ class TestDynamicPollster(base.BaseTestCase): sample = pollster.definitions.sample_extractor.generate_sample( pollster_sample, pollster.definitions.configurations, - manager=mock.Mock()) + manager=mock.Mock(), conf={}) self.assertEqual(1, sample.volume) self.assertEqual(2, len(sample.resource_metadata)) @@ -1127,7 +1596,7 @@ class TestDynamicPollster(base.BaseTestCase): sample = pollster.definitions.sample_extractor.generate_sample( pollster_sample, pollster.definitions.configurations, - manager=mock.Mock()) + manager=mock.Mock(), conf={}) self.assertEqual(1, sample.volume) self.assertEqual(3, len(sample.resource_metadata)) @@ -1150,7 +1619,7 @@ class TestDynamicPollster(base.BaseTestCase): sample = pollster.definitions.sample_extractor.generate_sample( pollster_sample, pollster.definitions.configurations, - manager=mock.Mock()) + manager=mock.Mock(), conf={}) self.assertEqual(1, sample.volume) self.assertEqual(3, len(sample.resource_metadata)) |