diff options
author | Alexey Yelistratov <ayelistratov@mirantis.com> | 2016-04-27 18:59:59 +0300 |
---|---|---|
committer | Alexey Yelistratov <ayelistratov@mirantis.com> | 2016-07-13 15:14:29 +0000 |
commit | a23d829dba6281148ffe37985094560ad7bca8d9 (patch) | |
tree | 26ff6a5e40cca36688f0ef08d36b1877a4c0127e | |
parent | 10f48b9cc86e44e4e22ec9ffdf3094f207e39120 (diff) | |
download | osprofiler-a23d829dba6281148ffe37985094560ad7bca8d9.tar.gz |
Add Ceilometer driver
Change-Id: I186ce1c33f56e62edc14edf099d83c545dd9c248
Spec: Multi backend support
-rw-r--r-- | osprofiler/cmd/commands.py | 43 | ||||
-rw-r--r-- | osprofiler/drivers/__init__.py | 1 | ||||
-rw-r--r-- | osprofiler/drivers/ceilometer.py | 81 | ||||
-rw-r--r-- | osprofiler/parsers/__init__.py | 0 | ||||
-rw-r--r-- | osprofiler/parsers/ceilometer.py | 138 | ||||
-rw-r--r-- | osprofiler/tests/cmd/test_shell.py | 66 | ||||
-rw-r--r-- | osprofiler/tests/drivers/test_ceilometer.py | 411 | ||||
-rw-r--r-- | osprofiler/tests/parsers/__init__.py | 0 | ||||
-rw-r--r-- | osprofiler/tests/parsers/test_ceilometer.py | 402 | ||||
-rw-r--r-- | test-requirements.txt | 4 |
10 files changed, 547 insertions, 599 deletions
diff --git a/osprofiler/cmd/commands.py b/osprofiler/cmd/commands.py index 800c008..2b6258f 100644 --- a/osprofiler/cmd/commands.py +++ b/osprofiler/cmd/commands.py @@ -17,8 +17,8 @@ import json import os from osprofiler.cmd import cliutils +from osprofiler.drivers import base from osprofiler import exc -from osprofiler.parsers import ceilometer as ceiloparser class BaseCommand(object): @@ -29,6 +29,9 @@ class TraceCommands(BaseCommand): group_name = "trace" @cliutils.arg("trace", help="File with trace or trace id") + @cliutils.arg("--connection-string", dest="conn_str", + default="ceilometer://", + help="storage driver's connection string") @cliutils.arg("--json", dest="use_json", action="store_true", help="show trace in JSON") @cliutils.arg("--html", dest="use_html", action="store_true", @@ -43,28 +46,11 @@ class TraceCommands(BaseCommand): trace = json.load(open(args.trace)) else: try: - import ceilometerclient.client - import ceilometerclient.exc - import ceilometerclient.shell - except ImportError: - raise ImportError( - "To use this command, you should install " - "'ceilometerclient' manually. Use command:\n " - "'pip install ceilometerclient'.") - try: - client = ceilometerclient.client.get_client( - args.ceilometer_api_version, **args.__dict__) - notifications = ceiloparser.get_notifications( - client, args.trace) + engine = base.get_driver(args.conn_str, **args.__dict__) except Exception as e: - if hasattr(e, "http_status") and e.http_status == 401: - msg = "Invalid OpenStack Identity credentials." - else: - msg = "Something has gone wrong. See logs for more details" - raise exc.CommandError(msg) + raise exc.CommandError(e.message) - if notifications: - trace = ceiloparser.parse_notifications(notifications) + trace = engine.get_report(args.trace) if not trace: msg = ("Trace with UUID %s not found. " @@ -76,13 +62,24 @@ class TraceCommands(BaseCommand): % args.trace) raise exc.CommandError(msg) + # NOTE(ayelistratov): Ceilometer translates datetime objects to + # strings, other drivers store this data in ISO Date format. + # Since datetime.datetime is not JSON serializable by default, + # this method will handle that. + def datetime_json_serialize(obj): + if hasattr(obj, "isoformat"): + return obj.isoformat() + else: + return obj + if args.use_json: - output = json.dumps(trace) + output = json.dumps(trace, default=datetime_json_serialize) elif args.use_html: with open(os.path.join(os.path.dirname(__file__), "template.html")) as html_template: output = html_template.read().replace( - "$DATA", json.dumps(trace, indent=2)) + "$DATA", json.dumps(trace, indent=2, + default=datetime_json_serialize)) else: raise exc.CommandError("You should choose one of the following " "output-formats: --json or --html.") diff --git a/osprofiler/drivers/__init__.py b/osprofiler/drivers/__init__.py index c89cabb..30f56e8 100644 --- a/osprofiler/drivers/__init__.py +++ b/osprofiler/drivers/__init__.py @@ -1,2 +1,3 @@ from osprofiler.drivers import base # noqa +from osprofiler.drivers import ceilometer # noqa from osprofiler.drivers import messaging # noqa diff --git a/osprofiler/drivers/ceilometer.py b/osprofiler/drivers/ceilometer.py new file mode 100644 index 0000000..85c2b55 --- /dev/null +++ b/osprofiler/drivers/ceilometer.py @@ -0,0 +1,81 @@ +# Copyright 2016 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from osprofiler.drivers import base +from osprofiler import exc + + +class Ceilometer(base.Driver): + def __init__(self, connection_str, **kwargs): + """Driver receiving profiled information from ceilometer.""" + super(Ceilometer, self).__init__(connection_str) + try: + import ceilometerclient.client + import ceilometerclient.shell + except ImportError: + raise exc.CommandError( + "To use this command, you should install " + "'ceilometerclient' manually. Use command:\n " + "'pip install python-ceilometerclient'.") + + try: + self.client = ceilometerclient.client.get_client( + kwargs["ceilometer_api_version"], **kwargs) + except Exception as e: + if hasattr(e, "http_status") and e.http_status == 401: + msg = "Invalid OpenStack Identity credentials." + else: + msg = ("Something has gone wrong. See ceilometer logs " + "for more details") + raise exc.CommandError(msg) + + @classmethod + def get_name(cls): + return "ceilometer" + + def get_report(self, base_id): + """Retrieves and parses notification from ceilometer. + + :param base_id: Base id of trace elements. + """ + + _filter = [{"field": "base_id", "op": "eq", "value": base_id}] + + # limit is hardcoded in this code state. Later that will be changed via + # connection string usage + notifications = [n.to_dict() + for n in self.client.events.list(_filter, + limit=100000)] + + for n in notifications: + traits = n["traits"] + + def find_field(f_name): + return [t["value"] for t in traits if t["name"] == f_name][0] + + trace_id = find_field("trace_id") + parent_id = find_field("parent_id") + name = find_field("name") + project = find_field("project") + service = find_field("service") + host = find_field("host") + timestamp = find_field("timestamp") + + payload = n.get("raw", {}).get("payload", {}) + + self._append_results(trace_id, parent_id, name, project, service, + host, timestamp, payload) + + return self._parse_results() diff --git a/osprofiler/parsers/__init__.py b/osprofiler/parsers/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/osprofiler/parsers/__init__.py +++ /dev/null diff --git a/osprofiler/parsers/ceilometer.py b/osprofiler/parsers/ceilometer.py deleted file mode 100644 index b267979..0000000 --- a/osprofiler/parsers/ceilometer.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2014 Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime - - -def _build_tree(nodes): - """Builds the tree (forest) data structure based on the list of nodes. - - Works in O(n). - - :param nodes: list of nodes, where each node is a dictionary with fields - "parent_id", "trace_id", "info" - :returns: list of top level ("root") nodes in form of dictionaries, - each containing the "info" and "children" fields, where - "children" is the list of child nodes ("children" will be - empty for leafs) - """ - - tree = [] - - for trace_id in nodes: - node = nodes[trace_id] - node.setdefault("children", []) - parent_id = node["parent_id"] - if parent_id in nodes: - nodes[parent_id].setdefault("children", []) - nodes[parent_id]["children"].append(node) - else: - tree.append(node) # no parent => top-level node - - for node in nodes: - nodes[node]["children"].sort(key=lambda x: x["info"]["started"]) - - return sorted(tree, key=lambda x: x["info"]["started"]) - - -def parse_notifications(notifications): - """Parse & builds tree structure from list of ceilometer notifications.""" - - result = {} - started_at = 0 - finished_at = 0 - - for n in notifications: - traits = n["traits"] - - def find_field(f_name): - return [t["value"] for t in traits if t["name"] == f_name][0] - - trace_id = find_field("trace_id") - parent_id = find_field("parent_id") - name = find_field("name") - project = find_field("project") - service = find_field("service") - host = find_field("host") - timestamp = find_field("timestamp") - - timestamp = datetime.datetime.strptime(timestamp, - "%Y-%m-%dT%H:%M:%S.%f") - - if trace_id not in result: - result[trace_id] = { - "info": { - "name": name.split("-")[0], - "project": project, - "service": service, - "host": host, - }, - "trace_id": trace_id, - "parent_id": parent_id, - } - - result[trace_id]["info"]["meta.raw_payload.%s" % name] = n.get( - "raw", {}).get("payload", {}) - - if name.endswith("stop"): - result[trace_id]["info"]["finished"] = timestamp - else: - result[trace_id]["info"]["started"] = timestamp - - if not started_at or started_at > timestamp: - started_at = timestamp - - if not finished_at or finished_at < timestamp: - finished_at = timestamp - - def msec(dt): - # NOTE(boris-42): Unfortunately this is the simplest way that works in - # py26 and py27 - microsec = (dt.microseconds + (dt.seconds + dt.days * 24 * 3600) * 1e6) - return int(microsec / 1000.0) - - for r in result.values(): - # NOTE(boris-42): We are not able to guarantee that ceilometer consumed - # all messages => so we should at make duration 0ms. - if "started" not in r["info"]: - r["info"]["started"] = r["info"]["finished"] - if "finished" not in r["info"]: - r["info"]["finished"] = r["info"]["started"] - - r["info"]["started"] = msec(r["info"]["started"] - started_at) - r["info"]["finished"] = msec(r["info"]["finished"] - started_at) - - return { - "info": { - "name": "total", - "started": 0, - "finished": msec(finished_at - started_at) if started_at else 0 - }, - "children": _build_tree(result) - } - - -def get_notifications(ceilometer, base_id): - """Retrieves and parses notification from ceilometer. - - :param ceilometer: Initialized ceilometer client. - :param base_id: Base id of trace elements. - """ - - _filter = [{"field": "base_id", "op": "eq", "value": base_id}] - # limit is hardcoded in this code state. Later that will be changed via - # connection string usage - return [n.to_dict() - for n in ceilometer.events.list(_filter, limit=100000)] diff --git a/osprofiler/tests/cmd/test_shell.py b/osprofiler/tests/cmd/test_shell.py index a6b6dd0..fa79c27 100644 --- a/osprofiler/tests/cmd/test_shell.py +++ b/osprofiler/tests/cmd/test_shell.py @@ -52,7 +52,7 @@ class ShellTestCase(test.TestCase): self.ceiloclient = mock.MagicMock() sys.modules["ceilometerclient"] = self.ceiloclient self.addCleanup(sys.modules.pop, "ceilometerclient", None) - ceilo_modules = ["client", "exc", "shell"] + ceilo_modules = ["client", "shell"] for module in ceilo_modules: sys.modules["ceilometerclient.%s" % module] = getattr( self.ceiloclient, module) @@ -80,7 +80,7 @@ class ShellTestCase(test.TestCase): self.assertEqual(str(actual_error), expected_message) else: raise ValueError( - "Expected: `osprofiler.cmd.exc.CommandError` is raised with " + "Expected: `osprofiler.exc.CommandError` is raised with " "message: '%s'." % expected_message) def test_username_is_not_presented(self): @@ -116,14 +116,15 @@ class ShellTestCase(test.TestCase): "env[OS_USER_DOMAIN_ID]") self._test_with_command_error("trace show fake-uuid", msg) - def test_trace_show_ceilometrclient_is_missed(self): + def test_trace_show_ceilometerclient_is_missed(self): sys.modules["ceilometerclient"] = None sys.modules["ceilometerclient.client"] = None - sys.modules["ceilometerclient.exc"] = None sys.modules["ceilometerclient.shell"] = None - self.assertRaises(ImportError, shell.main, - "trace show fake_uuid".split()) + msg = ("To use this command, you should install " + "'ceilometerclient' manually. Use command:\n " + "'pip install python-ceilometerclient'.") + self._test_with_command_error("trace show fake-uuid", msg) def test_trace_show_unauthorized(self): class FakeHTTPUnauthorized(Exception): @@ -139,18 +140,17 @@ class ShellTestCase(test.TestCase): pass self.ceiloclient.client.get_client.side_effect = FakeException - msg = "Something has gone wrong. See logs for more details" + msg = "Something has gone wrong. See ceilometer logs for more details" self._test_with_command_error("trace show fake_id", msg) - @mock.patch("osprofiler.parsers.ceilometer.get_notifications") - @mock.patch("osprofiler.parsers.ceilometer.parse_notifications") - def test_trace_show_no_selected_format(self, mock_notifications, mock_get): + @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report") + def test_trace_show_no_selected_format(self, mock_get): mock_get.return_value = "some_notificatios" msg = ("You should choose one of the following output-formats: " "--json or --html.") self._test_with_command_error("trace show fake_id", msg) - @mock.patch("osprofiler.parsers.ceilometer.get_notifications") + @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report") def test_trace_show_trace_id_not_found(self, mock_get): mock_get.return_value = None @@ -165,29 +165,26 @@ class ShellTestCase(test.TestCase): self._test_with_command_error("trace show %s" % fake_trace_id, msg) @mock.patch("sys.stdout", six.StringIO()) - @mock.patch("osprofiler.parsers.ceilometer.get_notifications") - @mock.patch("osprofiler.parsers.ceilometer.parse_notifications") - def test_trace_show_in_json(self, mock_notifications, mock_get): - mock_get.return_value = "some notification" + @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report") + def test_trace_show_in_json(self, mock_get): notifications = { "info": { "started": 0, "finished": 0, "name": "total"}, "children": []} - mock_notifications.return_value = notifications + + mock_get.return_value = notifications self.run_command("trace show fake_id --json") self.assertEqual("%s\n" % json.dumps(notifications), sys.stdout.getvalue()) @mock.patch("sys.stdout", six.StringIO()) - @mock.patch("osprofiler.parsers.ceilometer.get_notifications") - @mock.patch("osprofiler.parsers.ceilometer.parse_notifications") - def test_trace_show_in_html(self, mock_notifications, mock_get): - mock_get.return_value = "some notification" - + @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report") + def test_trace_show_in_html(self, mock_get): notifications = { "info": { "started": 0, "finished": 0, "name": "total"}, "children": []} - mock_notifications.return_value = notifications + + mock_get.return_value = notifications # NOTE(akurilin): to simplify assert statement, html-template should be # replaced. @@ -202,24 +199,23 @@ class ShellTestCase(test.TestCase): with mock.patch("osprofiler.cmd.commands.open", mock.mock_open(read_data=html_template), create=True): self.run_command("trace show fake_id --html") - self.assertEqual("A long time ago in a galaxy far, far away..." - " some_data = %s" - "It is a period of civil war. Rebel" - "spaceships, striking from a hidden" - "base, have won their first victory" - "against the evil Galactic Empire." - "\n" % json.dumps(notifications, indent=2), - sys.stdout.getvalue()) + self.assertEqual("A long time ago in a galaxy far, far away..." + " some_data = %s" + "It is a period of civil war. Rebel" + "spaceships, striking from a hidden" + "base, have won their first victory" + "against the evil Galactic Empire." + "\n" % json.dumps(notifications, indent=2), + sys.stdout.getvalue()) @mock.patch("sys.stdout", six.StringIO()) - @mock.patch("osprofiler.parsers.ceilometer.get_notifications") - @mock.patch("osprofiler.parsers.ceilometer.parse_notifications") - def test_trace_show_write_to_file(self, mock_notifications, mock_get): - mock_get.return_value = "some notification" + @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report") + def test_trace_show_write_to_file(self, mock_get): notifications = { "info": { "started": 0, "finished": 0, "name": "total"}, "children": []} - mock_notifications.return_value = notifications + + mock_get.return_value = notifications with mock.patch("osprofiler.cmd.commands.open", mock.mock_open(), create=True) as mock_open: diff --git a/osprofiler/tests/drivers/test_ceilometer.py b/osprofiler/tests/drivers/test_ceilometer.py new file mode 100644 index 0000000..6987ce7 --- /dev/null +++ b/osprofiler/tests/drivers/test_ceilometer.py @@ -0,0 +1,411 @@ +# Copyright 2016 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from osprofiler.drivers.ceilometer import Ceilometer +from osprofiler.tests import test + + +class CeilometerParserTestCase(test.TestCase): + def setUp(self): + super(CeilometerParserTestCase, self).setUp() + self.ceilometer = Ceilometer("ceilometer://", + ceilometer_api_version="2") + + def test_build_empty_tree(self): + self.assertEqual([], self.ceilometer._build_tree({})) + + def test_build_complex_tree(self): + test_input = { + "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}}, + "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}}, + "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}}, + "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}}, + "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}}, + "113": {"parent_id": "11", "trace_id": "113", + "info": {"started": 3}}, + "112": {"parent_id": "11", "trace_id": "112", + "info": {"started": 2}}, + "114": {"parent_id": "11", "trace_id": "114", + "info": {"started": 5}} + } + + expected_output = [ + { + "parent_id": "0", + "trace_id": "1", + "info": {"started": 0}, + "children": [ + { + "parent_id": "1", + "trace_id": "11", + "info": {"started": 1}, + "children": [ + {"parent_id": "11", "trace_id": "112", + "info": {"started": 2}, "children": []}, + {"parent_id": "11", "trace_id": "113", + "info": {"started": 3}, "children": []}, + {"parent_id": "11", "trace_id": "114", + "info": {"started": 5}, "children": []} + ] + } + ] + }, + { + "parent_id": "0", + "trace_id": "2", + "info": {"started": 1}, + "children": [ + {"parent_id": "2", "trace_id": "21", + "info": {"started": 6}, "children": []}, + {"parent_id": "2", "trace_id": "22", + "info": {"started": 7}, "children": []} + ] + } + ] + + result = self.ceilometer._build_tree(test_input) + self.assertEqual(expected_output, result) + + def test_get_report_empty(self): + self.ceilometer.client = mock.MagicMock() + self.ceilometer.client.events.list.return_value = [] + + expected = { + "info": { + "name": "total", + "started": 0, + "finished": None + }, + "children": [] + } + + base_id = "10" + self.assertEqual(expected, self.ceilometer.get_report(base_id)) + + def test_get_report(self): + self.ceilometer.client = mock.MagicMock() + results = [mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), + mock.MagicMock(), mock.MagicMock()] + + self.ceilometer.client.events.list.return_value = results + results[0].to_dict.return_value = { + "traits": [ + { + "type": "string", + "name": "base_id", + "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + { + "type": "string", + "name": "host", + "value": "ubuntu" + }, + { + "type": "string", + "name": "method", + "value": "POST" + }, + { + "type": "string", + "name": "name", + "value": "wsgi-start" + }, + { + "type": "string", + "name": "parent_id", + "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + { + "type": "string", + "name": "project", + "value": "keystone" + }, + { + "type": "string", + "name": "service", + "value": "main" + }, + { + "type": "string", + "name": "timestamp", + "value": "2015-12-23T14:02:22.338776" + }, + { + "type": "string", + "name": "trace_id", + "value": "06320327-2c2c-45ae-923a-515de890276a" + } + ], + "raw": {}, + "generated": "2015-12-23T10:41:38.415793", + "event_type": "profiler.main", + "message_id": "65fc1553-3082-4a6f-9d1e-0e3183f57a47"} + + results[1].to_dict.return_value = { + "traits": + [ + { + "type": "string", + "name": "base_id", + "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + { + "type": "string", + "name": "host", + "value": "ubuntu" + }, + { + "type": "string", + "name": "name", + "value": "wsgi-stop" + }, + { + "type": "string", + "name": "parent_id", + "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + { + "type": "string", + "name": "project", + "value": "keystone" + }, + { + "type": "string", + "name": "service", + "value": "main" + }, + { + "type": "string", + "name": "timestamp", + "value": "2015-12-23T14:02:22.380405" + }, + { + "type": "string", + "name": "trace_id", + "value": "016c97fd-87f3-40b2-9b55-e431156b694b" + } + ], + "raw": {}, + "generated": "2015-12-23T10:41:38.406052", + "event_type": "profiler.main", + "message_id": "3256d9f1-48ba-4ac5-a50b-64fa42c6e264"} + + results[2].to_dict.return_value = { + "traits": + [ + { + "type": "string", + "name": "base_id", + "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + { + "type": "string", + "name": "db.params", + "value": "[]" + }, + { + "type": "string", + "name": "db.statement", + "value": "SELECT 1" + }, + { + "type": "string", + "name": "host", + "value": "ubuntu" + }, + { + "type": "string", + "name": "name", + "value": "db-start" + }, + { + "type": "string", + "name": "parent_id", + "value": "06320327-2c2c-45ae-923a-515de890276a" + }, + { + "type": "string", + "name": "project", + "value": "keystone" + }, + { + "type": "string", + "name": "service", + "value": "main" + }, + { + "type": "string", + "name": "timestamp", + "value": "2015-12-23T14:02:22.395365" + }, + { + "type": "string", + "name": "trace_id", + "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a" + } + ], + "raw": {}, + "generated": "2015-12-23T10:41:38.984161", + "event_type": "profiler.main", + "message_id": "60368aa4-16f0-4f37-a8fb-89e92fdf36ff"} + + results[3].to_dict.return_value = { + "traits": + [ + { + "type": "string", + "name": "base_id", + "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + { + "type": "string", + "name": "host", + "value": "ubuntu" + }, + { + "type": "string", + "name": "name", + "value": "db-stop" + }, + { + "type": "string", + "name": "parent_id", + "value": "06320327-2c2c-45ae-923a-515de890276a" + }, + { + "type": "string", + "name": "project", + "value": "keystone" + }, + { + "type": "string", + "name": "service", + "value": "main" + }, + { + "type": "string", + "name": "timestamp", + "value": "2015-12-23T14:02:22.415486" + }, + { + "type": "string", + "name": "trace_id", + "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a" + } + ], + "raw": {}, + "generated": "2015-12-23T10:41:39.019378", + "event_type": "profiler.main", + "message_id": "3fbeb339-55c5-4f28-88e4-15bee251dd3d"} + + results[4].to_dict.return_value = { + "traits": + [ + { + "type": "string", + "name": "base_id", + "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + { + "type": "string", + "name": "host", + "value": "ubuntu" + }, + { + "type": "string", + "name": "method", + "value": "GET" + }, + { + "type": "string", + "name": "name", + "value": "wsgi-start" + }, + { + "type": "string", + "name": "parent_id", + "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + { + "type": "string", + "name": "project", + "value": "keystone" + }, + { + "type": "string", + "name": "service", + "value": "main" + }, + { + "type": "string", + "name": "timestamp", + "value": "2015-12-23T14:02:22.427444" + }, + { + "type": "string", + "name": "trace_id", + "value": "016c97fd-87f3-40b2-9b55-e431156b694b" + } + ], + "raw": {}, + "generated": "2015-12-23T10:41:38.360409", + "event_type": "profiler.main", + "message_id": "57b971a9-572f-4f29-9838-3ed2564c6b5b"} + + expected = {"children": [ + {"children": [{"children": [], + "info": {"finished": 76, + "host": "ubuntu", + "meta.raw_payload.db-start": {}, + "meta.raw_payload.db-stop": {}, + "name": "db", + "project": "keystone", + "service": "main", + "started": 56}, + "parent_id": "06320327-2c2c-45ae-923a-515de890276a", + "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"} + ], + "info": {"finished": 0, + "host": "ubuntu", + "meta.raw_payload.wsgi-start": {}, + "name": "wsgi", + "project": "keystone", + "service": "main", + "started": 0}, + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "trace_id": "06320327-2c2c-45ae-923a-515de890276a"}, + {"children": [], + "info": {"finished": 41, + "host": "ubuntu", + "meta.raw_payload.wsgi-start": {}, + "meta.raw_payload.wsgi-stop": {}, + "name": "wsgi", + "project": "keystone", + "service": "main", + "started": 88}, + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}], + "info": {"finished": 88, "name": "total", "started": 0}} + + base_id = "10" + + result = self.ceilometer.get_report(base_id) + + expected_filter = [{"field": "base_id", "op": "eq", "value": base_id}] + self.ceilometer.client.events.list.assert_called_once_with( + expected_filter, limit=100000) + self.assertEqual(expected, result) diff --git a/osprofiler/tests/parsers/__init__.py b/osprofiler/tests/parsers/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/osprofiler/tests/parsers/__init__.py +++ /dev/null diff --git a/osprofiler/tests/parsers/test_ceilometer.py b/osprofiler/tests/parsers/test_ceilometer.py deleted file mode 100644 index 103f244..0000000 --- a/osprofiler/tests/parsers/test_ceilometer.py +++ /dev/null @@ -1,402 +0,0 @@ -# Copyright 2014 Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import mock - -from osprofiler.parsers import ceilometer -from osprofiler.tests import test - - -class CeilometerParserTestCase(test.TestCase): - def test_build_empty_tree(self): - self.assertEqual(ceilometer._build_tree({}), []) - - def test_build_complex_tree(self): - test_input = { - "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}}, - "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}}, - "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}}, - "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}}, - "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}}, - "113": {"parent_id": "11", "trace_id": "113", - "info": {"started": 3}}, - "112": {"parent_id": "11", "trace_id": "112", - "info": {"started": 2}}, - "114": {"parent_id": "11", "trace_id": "114", - "info": {"started": 5}} - } - - expected_output = [ - { - "parent_id": "0", - "trace_id": "1", - "info": {"started": 0}, - "children": [ - { - "parent_id": "1", - "trace_id": "11", - "info": {"started": 1}, - "children": [ - {"parent_id": "11", "trace_id": "112", - "info": {"started": 2}, "children": []}, - {"parent_id": "11", "trace_id": "113", - "info": {"started": 3}, "children": []}, - {"parent_id": "11", "trace_id": "114", - "info": {"started": 5}, "children": []} - ] - } - ] - }, - { - "parent_id": "0", - "trace_id": "2", - "info": {"started": 1}, - "children": [ - {"parent_id": "2", "trace_id": "21", - "info": {"started": 6}, "children": []}, - {"parent_id": "2", "trace_id": "22", - "info": {"started": 7}, "children": []} - ] - } - ] - - self.assertEqual(ceilometer._build_tree(test_input), expected_output) - - def test_parse_notifications_empty(self): - expected = { - "info": { - "name": "total", - "started": 0, - "finished": 0 - }, - "children": [] - } - self.assertEqual(ceilometer.parse_notifications([]), expected) - - def test_parse_notifications(self): - events = [ - { - "traits": [ - { - "type": "string", - "name": "base_id", - "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" - }, - { - "type": "string", - "name": "host", - "value": "ubuntu" - }, - { - "type": "string", - "name": "method", - "value": "POST" - }, - { - "type": "string", - "name": "name", - "value": "wsgi-start" - }, - { - "type": "string", - "name": "parent_id", - "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" - }, - { - "type": "string", - "name": "project", - "value": "keystone" - }, - { - "type": "string", - "name": "service", - "value": "main" - }, - { - "type": "string", - "name": "timestamp", - "value": "2015-12-23T14:02:22.338776" - }, - { - "type": "string", - "name": "trace_id", - "value": "06320327-2c2c-45ae-923a-515de890276a" - } - ], - "raw": {}, - "generated": "2015-12-23T10:41:38.415793", - "event_type": "profiler.main", - "message_id": "65fc1553-3082-4a6f-9d1e-0e3183f57a47"}, - { - "traits": - [ - { - "type": "string", - "name": "base_id", - "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" - }, - { - "type": "string", - "name": "host", - "value": "ubuntu" - }, - { - "type": "string", - "name": "name", - "value": "wsgi-stop" - }, - { - "type": "string", - "name": "parent_id", - "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" - }, - { - "type": "string", - "name": "project", - "value": "keystone" - }, - { - "type": "string", - "name": "service", - "value": "main" - }, - { - "type": "string", - "name": "timestamp", - "value": "2015-12-23T14:02:22.380405" - }, - { - "type": "string", - "name": "trace_id", - "value": "016c97fd-87f3-40b2-9b55-e431156b694b" - } - ], - "raw": {}, - "generated": "2015-12-23T10:41:38.406052", - "event_type": "profiler.main", - "message_id": "3256d9f1-48ba-4ac5-a50b-64fa42c6e264"}, - { - "traits": - [ - { - "type": "string", - "name": "base_id", - "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" - }, - { - "type": "string", - "name": "db.params", - "value": "[]" - }, - { - "type": "string", - "name": "db.statement", - "value": "SELECT 1" - }, - { - "type": "string", - "name": "host", - "value": "ubuntu" - }, - { - "type": "string", - "name": "name", - "value": "db-start" - }, - { - "type": "string", - "name": "parent_id", - "value": "06320327-2c2c-45ae-923a-515de890276a" - }, - { - "type": "string", - "name": "project", - "value": "keystone" - }, - { - "type": "string", - "name": "service", - "value": "main" - }, - { - "type": "string", - "name": "timestamp", - "value": "2015-12-23T14:02:22.395365" - }, - { - "type": "string", - "name": "trace_id", - "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a" - } - ], - "raw": {}, - "generated": "2015-12-23T10:41:38.984161", - "event_type": "profiler.main", - "message_id": "60368aa4-16f0-4f37-a8fb-89e92fdf36ff" - }, - { - "traits": - [ - { - "type": "string", - "name": "base_id", - "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" - }, - { - "type": "string", - "name": "host", - "value": "ubuntu" - }, - { - "type": "string", - "name": "name", - "value": "db-stop" - }, - { - "type": "string", - "name": "parent_id", - "value": "06320327-2c2c-45ae-923a-515de890276a" - }, - { - "type": "string", - "name": "project", - "value": "keystone" - }, - { - "type": "string", - "name": "service", - "value": "main" - }, - { - "type": "string", - "name": "timestamp", - "value": "2015-12-23T14:02:22.415486" - }, - { - "type": "string", - "name": "trace_id", - "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a" - } - ], - "raw": {}, - "generated": "2015-12-23T10:41:39.019378", - "event_type": "profiler.main", - "message_id": "3fbeb339-55c5-4f28-88e4-15bee251dd3d" - }, - { - "traits": - [ - { - "type": "string", - "name": "base_id", - "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" - }, - { - "type": "string", - "name": "host", - "value": "ubuntu" - }, - { - "type": "string", - "name": "method", - "value": "GET" - }, - { - "type": "string", - "name": "name", - "value": "wsgi-start" - }, - { - "type": "string", - "name": "parent_id", - "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" - }, - { - "type": "string", - "name": "project", - "value": "keystone" - }, - { - "type": "string", - "name": "service", - "value": "main" - }, - { - "type": "string", - "name": "timestamp", - "value": "2015-12-23T14:02:22.427444" - }, - { - "type": "string", - "name": "trace_id", - "value": "016c97fd-87f3-40b2-9b55-e431156b694b" - } - ], - "raw": {}, - "generated": "2015-12-23T10:41:38.360409", - "event_type": "profiler.main", - "message_id": "57b971a9-572f-4f29-9838-3ed2564c6b5b" - } - ] - - expected = {"children": [ - {"children": [{"children": [], - "info": {"finished": 76, - "host": "ubuntu", - "meta.raw_payload.db-start": {}, - "meta.raw_payload.db-stop": {}, - "name": "db", - "project": "keystone", - "service": "main", - "started": 56}, - "parent_id": "06320327-2c2c-45ae-923a-515de890276a", - "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"} - ], - "info": {"finished": 0, - "host": "ubuntu", - "meta.raw_payload.wsgi-start": {}, - "name": "wsgi", - "project": "keystone", - "service": "main", - "started": 0}, - "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", - "trace_id": "06320327-2c2c-45ae-923a-515de890276a"}, - {"children": [], - "info": {"finished": 41, - "host": "ubuntu", - "meta.raw_payload.wsgi-start": {}, - "meta.raw_payload.wsgi-stop": {}, - "name": "wsgi", - "project": "keystone", - "service": "main", - "started": 88}, - "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", - "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}], - "info": {"finished": 88, "name": "total", "started": 0}} - - self.assertEqual(expected, ceilometer.parse_notifications(events)) - - def test_get_notifications(self): - mock_ceil_client = mock.MagicMock() - results = [mock.MagicMock(), mock.MagicMock()] - mock_ceil_client.events.list.return_value = results - base_id = "10" - - result = ceilometer.get_notifications(mock_ceil_client, base_id) - - expected_filter = [{"field": "base_id", "op": "eq", "value": base_id}] - mock_ceil_client.events.list.assert_called_once_with(expected_filter, - limit=100000) - self.assertEqual(result, [results[0].to_dict(), results[1].to_dict()]) diff --git a/test-requirements.txt b/test-requirements.txt index c945d16..2dff41b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,4 +10,6 @@ oslosphinx>=2.5.0,!=3.4.0 # Apache-2.0 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 # Bandit security code scanner -bandit>=0.17.3 # Apache-2.0
\ No newline at end of file +bandit>=0.17.3 # Apache-2.0 + +python-ceilometerclient>=2.2.1 # Apache-2.0 |