Merge "Add Ceilometer driver"

osprofiler/cmd/commands.py

@@ -17,8 +17,8 @@ import json
 import os
 
 from osprofiler.cmd import cliutils
+from osprofiler.drivers import base
 from osprofiler import exc
-from osprofiler.parsers import ceilometer as ceiloparser
 
 
 class BaseCommand(object):
@@ -29,6 +29,9 @@ class TraceCommands(BaseCommand):
     group_name = "trace"
 
     @cliutils.arg("trace", help="File with trace or trace id")
+    @cliutils.arg("--connection-string", dest="conn_str",
+                  default="ceilometer://",
+                  help="storage driver's connection string")
     @cliutils.arg("--json", dest="use_json", action="store_true",
                   help="show trace in JSON")
     @cliutils.arg("--html", dest="use_html", action="store_true",
@@ -43,28 +46,11 @@ class TraceCommands(BaseCommand):
             trace = json.load(open(args.trace))
         else:
             try:
-                import ceilometerclient.client
+                engine = base.get_driver(args.conn_str, **args.__dict__)
-                import ceilometerclient.exc
-                import ceilometerclient.shell
-            except ImportError:
-                raise ImportError(
-                    "To use this command, you should install "
-                    "'ceilometerclient' manually. Use command:\n "
-                    "'pip install ceilometerclient'.")
-            try:
-                client = ceilometerclient.client.get_client(
-                    args.ceilometer_api_version, **args.__dict__)
-                notifications = ceiloparser.get_notifications(
-                    client, args.trace)
             except Exception as e:
-                if hasattr(e, "http_status") and e.http_status == 401:
+                raise exc.CommandError(e.message)
-                    msg = "Invalid OpenStack Identity credentials."
-                else:
-                    msg = "Something has gone wrong. See logs for more details"
-                raise exc.CommandError(msg)
 
-            if notifications:
+            trace = engine.get_report(args.trace)
-                trace = ceiloparser.parse_notifications(notifications)
 
         if not trace:
             msg = ("Trace with UUID %s not found. "
@@ -76,13 +62,24 @@ class TraceCommands(BaseCommand):
                    % args.trace)
             raise exc.CommandError(msg)
 
+        # NOTE(ayelistratov): Ceilometer translates datetime objects to
+        # strings, other drivers store this data in ISO Date format.
+        # Since datetime.datetime is not JSON serializable by default,
+        # this method will handle that.
+        def datetime_json_serialize(obj):
+            if hasattr(obj, "isoformat"):
+                return obj.isoformat()
+            else:
+                return obj
+
         if args.use_json:
-            output = json.dumps(trace)
+            output = json.dumps(trace, default=datetime_json_serialize)
         elif args.use_html:
             with open(os.path.join(os.path.dirname(__file__),
                                    "template.html")) as html_template:
                 output = html_template.read().replace(
-                    "$DATA", json.dumps(trace, indent=2))
+                    "$DATA", json.dumps(trace, indent=2,
+                                        default=datetime_json_serialize))
         else:
             raise exc.CommandError("You should choose one of the following "
                                    "output-formats: --json or --html.")

osprofiler/drivers/__init__.py

@@ -1,3 +1,4 @@
 from osprofiler.drivers import base  # noqa
+from osprofiler.drivers import ceilometer  # noqa
 from osprofiler.drivers import messaging  # noqa
 from osprofiler.drivers import mongodb  # noqa

osprofiler/drivers/ceilometer.py

@@ -0,0 +1,81 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from osprofiler.drivers import base
+from osprofiler import exc
+
+
+class Ceilometer(base.Driver):
+    def __init__(self, connection_str, **kwargs):
+        """Driver receiving profiled information from ceilometer."""
+        super(Ceilometer, self).__init__(connection_str)
+        try:
+            import ceilometerclient.client
+            import ceilometerclient.shell
+        except ImportError:
+            raise exc.CommandError(
+                "To use this command, you should install "
+                "'ceilometerclient' manually. Use command:\n "
+                "'pip install python-ceilometerclient'.")
+
+        try:
+            self.client = ceilometerclient.client.get_client(
+                kwargs["ceilometer_api_version"], **kwargs)
+        except Exception as e:
+            if hasattr(e, "http_status") and e.http_status == 401:
+                msg = "Invalid OpenStack Identity credentials."
+            else:
+                msg = ("Something has gone wrong. See ceilometer logs "
+                       "for more details")
+            raise exc.CommandError(msg)
+
+    @classmethod
+    def get_name(cls):
+        return "ceilometer"
+
+    def get_report(self, base_id):
+        """Retrieves and parses notification from ceilometer.
+
+        :param base_id: Base id of trace elements.
+        """
+
+        _filter = [{"field": "base_id", "op": "eq", "value": base_id}]
+
+        # limit is hardcoded in this code state. Later that will be changed via
+        # connection string usage
+        notifications = [n.to_dict()
+                         for n in self.client.events.list(_filter,
+                                                          limit=100000)]
+
+        for n in notifications:
+            traits = n["traits"]
+
+            def find_field(f_name):
+                return [t["value"] for t in traits if t["name"] == f_name][0]
+
+            trace_id = find_field("trace_id")
+            parent_id = find_field("parent_id")
+            name = find_field("name")
+            project = find_field("project")
+            service = find_field("service")
+            host = find_field("host")
+            timestamp = find_field("timestamp")
+
+            payload = n.get("raw", {}).get("payload", {})
+
+            self._append_results(trace_id, parent_id, name, project, service,
+                                 host, timestamp, payload)
+
+        return self._parse_results()

osprofiler/parsers/ceilometer.py

@@ -1,138 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import datetime
-
-
-def _build_tree(nodes):
-    """Builds the tree (forest) data structure based on the list of nodes.
-
-   Works in O(n).
-
-   :param nodes: list of nodes, where each node is a dictionary with fields
-                 "parent_id", "trace_id", "info"
-   :returns: list of top level ("root") nodes in form of dictionaries,
-             each containing the "info" and "children" fields, where
-             "children" is the list of child nodes ("children" will be
-             empty for leafs)
-   """
-
-    tree = []
-
-    for trace_id in nodes:
-        node = nodes[trace_id]
-        node.setdefault("children", [])
-        parent_id = node["parent_id"]
-        if parent_id in nodes:
-            nodes[parent_id].setdefault("children", [])
-            nodes[parent_id]["children"].append(node)
-        else:
-            tree.append(node)  # no parent => top-level node
-
-    for node in nodes:
-        nodes[node]["children"].sort(key=lambda x: x["info"]["started"])
-
-    return sorted(tree, key=lambda x: x["info"]["started"])
-
-
-def parse_notifications(notifications):
-    """Parse & builds tree structure from list of ceilometer notifications."""
-
-    result = {}
-    started_at = 0
-    finished_at = 0
-
-    for n in notifications:
-        traits = n["traits"]
-
-        def find_field(f_name):
-            return [t["value"] for t in traits if t["name"] == f_name][0]
-
-        trace_id = find_field("trace_id")
-        parent_id = find_field("parent_id")
-        name = find_field("name")
-        project = find_field("project")
-        service = find_field("service")
-        host = find_field("host")
-        timestamp = find_field("timestamp")
-
-        timestamp = datetime.datetime.strptime(timestamp,
-                                               "%Y-%m-%dT%H:%M:%S.%f")
-
-        if trace_id not in result:
-            result[trace_id] = {
-                "info": {
-                    "name": name.split("-")[0],
-                    "project": project,
-                    "service": service,
-                    "host": host,
-                },
-                "trace_id": trace_id,
-                "parent_id": parent_id,
-            }
-
-        result[trace_id]["info"]["meta.raw_payload.%s" % name] = n.get(
-            "raw", {}).get("payload", {})
-
-        if name.endswith("stop"):
-            result[trace_id]["info"]["finished"] = timestamp
-        else:
-            result[trace_id]["info"]["started"] = timestamp
-
-        if not started_at or started_at > timestamp:
-            started_at = timestamp
-
-        if not finished_at or finished_at < timestamp:
-            finished_at = timestamp
-
-    def msec(dt):
-        # NOTE(boris-42): Unfortunately this is the simplest way that works in
-        #                 py26 and py27
-        microsec = (dt.microseconds + (dt.seconds + dt.days * 24 * 3600) * 1e6)
-        return int(microsec / 1000.0)
-
-    for r in result.values():
-        # NOTE(boris-42): We are not able to guarantee that ceilometer consumed
-        #                 all messages => so we should at make duration 0ms.
-        if "started" not in r["info"]:
-            r["info"]["started"] = r["info"]["finished"]
-        if "finished" not in r["info"]:
-            r["info"]["finished"] = r["info"]["started"]
-
-        r["info"]["started"] = msec(r["info"]["started"] - started_at)
-        r["info"]["finished"] = msec(r["info"]["finished"] - started_at)
-
-    return {
-        "info": {
-            "name": "total",
-            "started": 0,
-            "finished": msec(finished_at - started_at) if started_at else 0
-        },
-        "children": _build_tree(result)
-    }
-
-
-def get_notifications(ceilometer, base_id):
-    """Retrieves and parses notification from ceilometer.
-
-    :param ceilometer: Initialized ceilometer client.
-    :param base_id: Base id of trace elements.
-    """
-
-    _filter = [{"field": "base_id", "op": "eq", "value": base_id}]
-    # limit is hardcoded in this code state. Later that will be changed via
-    # connection string usage
-    return [n.to_dict()
-            for n in ceilometer.events.list(_filter, limit=100000)]

osprofiler/tests/cmd/test_shell.py

@@ -52,7 +52,7 @@ class ShellTestCase(test.TestCase):
         self.ceiloclient = mock.MagicMock()
         sys.modules["ceilometerclient"] = self.ceiloclient
         self.addCleanup(sys.modules.pop, "ceilometerclient", None)
-        ceilo_modules = ["client", "exc", "shell"]
+        ceilo_modules = ["client", "shell"]
         for module in ceilo_modules:
             sys.modules["ceilometerclient.%s" % module] = getattr(
                 self.ceiloclient, module)
@@ -80,7 +80,7 @@ class ShellTestCase(test.TestCase):
             self.assertEqual(str(actual_error), expected_message)
         else:
             raise ValueError(
-                "Expected: `osprofiler.cmd.exc.CommandError` is raised with "
+                "Expected: `osprofiler.exc.CommandError` is raised with "
                 "message: '%s'." % expected_message)
 
     def test_username_is_not_presented(self):
@@ -116,14 +116,15 @@ class ShellTestCase(test.TestCase):
                "env[OS_USER_DOMAIN_ID]")
         self._test_with_command_error("trace show fake-uuid", msg)
 
-    def test_trace_show_ceilometrclient_is_missed(self):
+    def test_trace_show_ceilometerclient_is_missed(self):
         sys.modules["ceilometerclient"] = None
         sys.modules["ceilometerclient.client"] = None
-        sys.modules["ceilometerclient.exc"] = None
         sys.modules["ceilometerclient.shell"] = None
 
-        self.assertRaises(ImportError, shell.main,
+        msg = ("To use this command, you should install "
-                          "trace show fake_uuid".split())
+               "'ceilometerclient' manually. Use command:\n "
+               "'pip install python-ceilometerclient'.")
+        self._test_with_command_error("trace show fake-uuid", msg)
 
     def test_trace_show_unauthorized(self):
         class FakeHTTPUnauthorized(Exception):
@@ -139,18 +140,17 @@ class ShellTestCase(test.TestCase):
             pass
 
         self.ceiloclient.client.get_client.side_effect = FakeException
-        msg = "Something has gone wrong. See logs for more details"
+        msg = "Something has gone wrong. See ceilometer logs for more details"
         self._test_with_command_error("trace show fake_id", msg)
 
-    @mock.patch("osprofiler.parsers.ceilometer.get_notifications")
+    @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report")
-    @mock.patch("osprofiler.parsers.ceilometer.parse_notifications")
+    def test_trace_show_no_selected_format(self, mock_get):
-    def test_trace_show_no_selected_format(self, mock_notifications, mock_get):
         mock_get.return_value = "some_notificatios"
         msg = ("You should choose one of the following output-formats: "
                "--json or --html.")
         self._test_with_command_error("trace show fake_id", msg)
 
-    @mock.patch("osprofiler.parsers.ceilometer.get_notifications")
+    @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report")
     def test_trace_show_trace_id_not_found(self, mock_get):
         mock_get.return_value = None
 
@@ -165,29 +165,26 @@ class ShellTestCase(test.TestCase):
         self._test_with_command_error("trace show %s" % fake_trace_id, msg)
 
     @mock.patch("sys.stdout", six.StringIO())
-    @mock.patch("osprofiler.parsers.ceilometer.get_notifications")
+    @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report")
-    @mock.patch("osprofiler.parsers.ceilometer.parse_notifications")
+    def test_trace_show_in_json(self, mock_get):
-    def test_trace_show_in_json(self, mock_notifications, mock_get):
-        mock_get.return_value = "some notification"
         notifications = {
             "info": {
                 "started": 0, "finished": 0, "name": "total"}, "children": []}
-        mock_notifications.return_value = notifications
+
+        mock_get.return_value = notifications
 
         self.run_command("trace show fake_id --json")
         self.assertEqual("%s\n" % json.dumps(notifications),
                          sys.stdout.getvalue())
 
     @mock.patch("sys.stdout", six.StringIO())
-    @mock.patch("osprofiler.parsers.ceilometer.get_notifications")
+    @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report")
-    @mock.patch("osprofiler.parsers.ceilometer.parse_notifications")
+    def test_trace_show_in_html(self, mock_get):
-    def test_trace_show_in_html(self, mock_notifications, mock_get):
-        mock_get.return_value = "some notification"
-
         notifications = {
             "info": {
                 "started": 0, "finished": 0, "name": "total"}, "children": []}
-        mock_notifications.return_value = notifications
+
+        mock_get.return_value = notifications
 
         # NOTE(akurilin): to simplify assert statement, html-template should be
         # replaced.
@@ -202,24 +199,23 @@ class ShellTestCase(test.TestCase):
         with mock.patch("osprofiler.cmd.commands.open",
                         mock.mock_open(read_data=html_template), create=True):
             self.run_command("trace show fake_id --html")
-        self.assertEqual("A long time ago in a galaxy far, far away..."
+            self.assertEqual("A long time ago in a galaxy far, far away..."
-                         "    some_data = %s"
+                             "    some_data = %s"
-                         "It is a period of civil war. Rebel"
+                             "It is a period of civil war. Rebel"
-                         "spaceships, striking from a hidden"
+                             "spaceships, striking from a hidden"
-                         "base, have won their first victory"
+                             "base, have won their first victory"
-                         "against the evil Galactic Empire."
+                             "against the evil Galactic Empire."
-                         "\n" % json.dumps(notifications, indent=2),
+                             "\n" % json.dumps(notifications, indent=2),
-                         sys.stdout.getvalue())
+                             sys.stdout.getvalue())
 
     @mock.patch("sys.stdout", six.StringIO())
-    @mock.patch("osprofiler.parsers.ceilometer.get_notifications")
+    @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report")
-    @mock.patch("osprofiler.parsers.ceilometer.parse_notifications")
+    def test_trace_show_write_to_file(self, mock_get):
-    def test_trace_show_write_to_file(self, mock_notifications, mock_get):
-        mock_get.return_value = "some notification"
         notifications = {
             "info": {
                 "started": 0, "finished": 0, "name": "total"}, "children": []}
-        mock_notifications.return_value = notifications
+
+        mock_get.return_value = notifications
 
         with mock.patch("osprofiler.cmd.commands.open",
                         mock.mock_open(), create=True) as mock_open:

osprofiler/tests/drivers/test_ceilometer.py

@@ -0,0 +1,411 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from osprofiler.drivers.ceilometer import Ceilometer
+from osprofiler.tests import test
+
+
+class CeilometerParserTestCase(test.TestCase):
+    def setUp(self):
+        super(CeilometerParserTestCase, self).setUp()
+        self.ceilometer = Ceilometer("ceilometer://",
+                                     ceilometer_api_version="2")
+
+    def test_build_empty_tree(self):
+        self.assertEqual([], self.ceilometer._build_tree({}))
+
+    def test_build_complex_tree(self):
+        test_input = {
+            "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}},
+            "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}},
+            "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}},
+            "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}},
+            "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}},
+            "113": {"parent_id": "11", "trace_id": "113",
+                    "info": {"started": 3}},
+            "112": {"parent_id": "11", "trace_id": "112",
+                    "info": {"started": 2}},
+            "114": {"parent_id": "11", "trace_id": "114",
+                    "info": {"started": 5}}
+        }
+
+        expected_output = [
+            {
+                "parent_id": "0",
+                "trace_id": "1",
+                "info": {"started": 0},
+                "children": [
+                    {
+                        "parent_id": "1",
+                        "trace_id": "11",
+                        "info": {"started": 1},
+                        "children": [
+                            {"parent_id": "11", "trace_id": "112",
+                             "info": {"started": 2}, "children": []},
+                            {"parent_id": "11", "trace_id": "113",
+                             "info": {"started": 3}, "children": []},
+                            {"parent_id": "11", "trace_id": "114",
+                             "info": {"started": 5}, "children": []}
+                        ]
+                    }
+                ]
+            },
+            {
+                "parent_id": "0",
+                "trace_id": "2",
+                "info": {"started": 1},
+                "children": [
+                    {"parent_id": "2", "trace_id": "21",
+                     "info": {"started": 6}, "children": []},
+                    {"parent_id": "2", "trace_id": "22",
+                     "info": {"started": 7}, "children": []}
+                ]
+            }
+        ]
+
+        result = self.ceilometer._build_tree(test_input)
+        self.assertEqual(expected_output, result)
+
+    def test_get_report_empty(self):
+        self.ceilometer.client = mock.MagicMock()
+        self.ceilometer.client.events.list.return_value = []
+
+        expected = {
+            "info": {
+                "name": "total",
+                "started": 0,
+                "finished": None
+            },
+            "children": []
+        }
+
+        base_id = "10"
+        self.assertEqual(expected, self.ceilometer.get_report(base_id))
+
+    def test_get_report(self):
+        self.ceilometer.client = mock.MagicMock()
+        results = [mock.MagicMock(), mock.MagicMock(), mock.MagicMock(),
+                   mock.MagicMock(), mock.MagicMock()]
+
+        self.ceilometer.client.events.list.return_value = results
+        results[0].to_dict.return_value = {
+            "traits": [
+                {
+                    "type": "string",
+                    "name": "base_id",
+                    "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+                },
+                {
+                    "type": "string",
+                    "name": "host",
+                    "value": "ubuntu"
+                },
+                {
+                    "type": "string",
+                    "name": "method",
+                    "value": "POST"
+                },
+                {
+                    "type": "string",
+                    "name": "name",
+                    "value": "wsgi-start"
+                },
+                {
+                    "type": "string",
+                    "name": "parent_id",
+                    "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+                },
+                {
+                    "type": "string",
+                    "name": "project",
+                    "value": "keystone"
+                },
+                {
+                    "type": "string",
+                    "name": "service",
+                    "value": "main"
+                },
+                {
+                    "type": "string",
+                    "name": "timestamp",
+                    "value": "2015-12-23T14:02:22.338776"
+                },
+                {
+                    "type": "string",
+                    "name": "trace_id",
+                    "value": "06320327-2c2c-45ae-923a-515de890276a"
+                }
+            ],
+            "raw": {},
+            "generated": "2015-12-23T10:41:38.415793",
+            "event_type": "profiler.main",
+            "message_id": "65fc1553-3082-4a6f-9d1e-0e3183f57a47"}
+
+        results[1].to_dict.return_value = {
+            "traits":
+                [
+                    {
+                        "type": "string",
+                        "name": "base_id",
+                        "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+                    },
+                    {
+                        "type": "string",
+                        "name": "host",
+                        "value": "ubuntu"
+                    },
+                    {
+                        "type": "string",
+                        "name": "name",
+                        "value": "wsgi-stop"
+                    },
+                    {
+                        "type": "string",
+                        "name": "parent_id",
+                        "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+                    },
+                    {
+                        "type": "string",
+                        "name": "project",
+                        "value": "keystone"
+                    },
+                    {
+                        "type": "string",
+                        "name": "service",
+                        "value": "main"
+                    },
+                    {
+                        "type": "string",
+                        "name": "timestamp",
+                        "value": "2015-12-23T14:02:22.380405"
+                    },
+                    {
+                        "type": "string",
+                        "name": "trace_id",
+                        "value": "016c97fd-87f3-40b2-9b55-e431156b694b"
+                    }
+                ],
+            "raw": {},
+            "generated": "2015-12-23T10:41:38.406052",
+            "event_type": "profiler.main",
+            "message_id": "3256d9f1-48ba-4ac5-a50b-64fa42c6e264"}
+
+        results[2].to_dict.return_value = {
+            "traits":
+                [
+                    {
+                        "type": "string",
+                        "name": "base_id",
+                        "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+                    },
+                    {
+                        "type": "string",
+                        "name": "db.params",
+                        "value": "[]"
+                    },
+                    {
+                        "type": "string",
+                        "name": "db.statement",
+                        "value": "SELECT 1"
+                    },
+                    {
+                        "type": "string",
+                        "name": "host",
+                        "value": "ubuntu"
+                    },
+                    {
+                        "type": "string",
+                        "name": "name",
+                        "value": "db-start"
+                    },
+                    {
+                        "type": "string",
+                        "name": "parent_id",
+                        "value": "06320327-2c2c-45ae-923a-515de890276a"
+                    },
+                    {
+                        "type": "string",
+                        "name": "project",
+                        "value": "keystone"
+                    },
+                    {
+                        "type": "string",
+                        "name": "service",
+                        "value": "main"
+                    },
+                    {
+                        "type": "string",
+                        "name": "timestamp",
+                        "value": "2015-12-23T14:02:22.395365"
+                    },
+                    {
+                        "type": "string",
+                        "name": "trace_id",
+                        "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"
+                    }
+                ],
+            "raw": {},
+            "generated": "2015-12-23T10:41:38.984161",
+            "event_type": "profiler.main",
+            "message_id": "60368aa4-16f0-4f37-a8fb-89e92fdf36ff"}
+
+        results[3].to_dict.return_value = {
+            "traits":
+                [
+                    {
+                        "type": "string",
+                        "name": "base_id",
+                        "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+                    },
+                    {
+                        "type": "string",
+                        "name": "host",
+                        "value": "ubuntu"
+                    },
+                    {
+                        "type": "string",
+                        "name": "name",
+                        "value": "db-stop"
+                    },
+                    {
+                        "type": "string",
+                        "name": "parent_id",
+                        "value": "06320327-2c2c-45ae-923a-515de890276a"
+                    },
+                    {
+                        "type": "string",
+                        "name": "project",
+                        "value": "keystone"
+                    },
+                    {
+                        "type": "string",
+                        "name": "service",
+                        "value": "main"
+                    },
+                    {
+                        "type": "string",
+                        "name": "timestamp",
+                        "value": "2015-12-23T14:02:22.415486"
+                    },
+                    {
+                        "type": "string",
+                        "name": "trace_id",
+                        "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"
+                    }
+                ],
+            "raw": {},
+            "generated": "2015-12-23T10:41:39.019378",
+            "event_type": "profiler.main",
+            "message_id": "3fbeb339-55c5-4f28-88e4-15bee251dd3d"}
+
+        results[4].to_dict.return_value = {
+            "traits":
+                [
+                    {
+                        "type": "string",
+                        "name": "base_id",
+                        "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+                    },
+                    {
+                        "type": "string",
+                        "name": "host",
+                        "value": "ubuntu"
+                    },
+                    {
+                        "type": "string",
+                        "name": "method",
+                        "value": "GET"
+                    },
+                    {
+                        "type": "string",
+                        "name": "name",
+                        "value": "wsgi-start"
+                    },
+                    {
+                        "type": "string",
+                        "name": "parent_id",
+                        "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+                    },
+                    {
+                        "type": "string",
+                        "name": "project",
+                        "value": "keystone"
+                    },
+                    {
+                        "type": "string",
+                        "name": "service",
+                        "value": "main"
+                    },
+                    {
+                        "type": "string",
+                        "name": "timestamp",
+                        "value": "2015-12-23T14:02:22.427444"
+                    },
+                    {
+                        "type": "string",
+                        "name": "trace_id",
+                        "value": "016c97fd-87f3-40b2-9b55-e431156b694b"
+                    }
+                ],
+            "raw": {},
+            "generated": "2015-12-23T10:41:38.360409",
+            "event_type": "profiler.main",
+            "message_id": "57b971a9-572f-4f29-9838-3ed2564c6b5b"}
+
+        expected = {"children": [
+            {"children": [{"children": [],
+                           "info": {"finished": 76,
+                                    "host": "ubuntu",
+                                    "meta.raw_payload.db-start": {},
+                                    "meta.raw_payload.db-stop": {},
+                                    "name": "db",
+                                    "project": "keystone",
+                                    "service": "main",
+                                    "started": 56},
+                           "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+                           "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"}
+                          ],
+             "info": {"finished": 0,
+                      "host": "ubuntu",
+                      "meta.raw_payload.wsgi-start": {},
+                      "name": "wsgi",
+                      "project": "keystone",
+                      "service": "main",
+                      "started": 0},
+             "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+             "trace_id": "06320327-2c2c-45ae-923a-515de890276a"},
+            {"children": [],
+             "info": {"finished": 41,
+                      "host": "ubuntu",
+                      "meta.raw_payload.wsgi-start": {},
+                      "meta.raw_payload.wsgi-stop": {},
+                      "name": "wsgi",
+                      "project": "keystone",
+                      "service": "main",
+                      "started": 88},
+             "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+             "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}],
+            "info": {"finished": 88, "name": "total", "started": 0}}
+
+        base_id = "10"
+
+        result = self.ceilometer.get_report(base_id)
+
+        expected_filter = [{"field": "base_id", "op": "eq", "value": base_id}]
+        self.ceilometer.client.events.list.assert_called_once_with(
+            expected_filter, limit=100000)
+        self.assertEqual(expected, result)

osprofiler/tests/parsers/test_ceilometer.py

@@ -1,402 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import mock
-
-from osprofiler.parsers import ceilometer
-from osprofiler.tests import test
-
-
-class CeilometerParserTestCase(test.TestCase):
-    def test_build_empty_tree(self):
-        self.assertEqual(ceilometer._build_tree({}), [])
-
-    def test_build_complex_tree(self):
-        test_input = {
-            "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}},
-            "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}},
-            "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}},
-            "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}},
-            "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}},
-            "113": {"parent_id": "11", "trace_id": "113",
-                    "info": {"started": 3}},
-            "112": {"parent_id": "11", "trace_id": "112",
-                    "info": {"started": 2}},
-            "114": {"parent_id": "11", "trace_id": "114",
-                    "info": {"started": 5}}
-        }
-
-        expected_output = [
-            {
-                "parent_id": "0",
-                "trace_id": "1",
-                "info": {"started": 0},
-                "children": [
-                    {
-                        "parent_id": "1",
-                        "trace_id": "11",
-                        "info": {"started": 1},
-                        "children": [
-                            {"parent_id": "11", "trace_id": "112",
-                             "info": {"started": 2}, "children": []},
-                            {"parent_id": "11", "trace_id": "113",
-                             "info": {"started": 3}, "children": []},
-                            {"parent_id": "11", "trace_id": "114",
-                             "info": {"started": 5}, "children": []}
-                        ]
-                    }
-                ]
-            },
-            {
-                "parent_id": "0",
-                "trace_id": "2",
-                "info": {"started": 1},
-                "children": [
-                    {"parent_id": "2", "trace_id": "21",
-                     "info": {"started": 6}, "children": []},
-                    {"parent_id": "2", "trace_id": "22",
-                     "info": {"started": 7}, "children": []}
-                ]
-            }
-        ]
-
-        self.assertEqual(ceilometer._build_tree(test_input), expected_output)
-
-    def test_parse_notifications_empty(self):
-        expected = {
-            "info": {
-                "name": "total",
-                "started": 0,
-                "finished": 0
-            },
-            "children": []
-        }
-        self.assertEqual(ceilometer.parse_notifications([]), expected)
-
-    def test_parse_notifications(self):
-        events = [
-            {
-                "traits": [
-                    {
-                        "type": "string",
-                        "name": "base_id",
-                        "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
-                    },
-                    {
-                        "type": "string",
-                        "name": "host",
-                        "value": "ubuntu"
-                    },
-                    {
-                        "type": "string",
-                        "name": "method",
-                        "value": "POST"
-                    },
-                    {
-                        "type": "string",
-                        "name": "name",
-                        "value": "wsgi-start"
-                    },
-                    {
-                        "type": "string",
-                        "name": "parent_id",
-                        "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
-                    },
-                    {
-                        "type": "string",
-                        "name": "project",
-                        "value": "keystone"
-                    },
-                    {
-                        "type": "string",
-                        "name": "service",
-                        "value": "main"
-                    },
-                    {
-                        "type": "string",
-                        "name": "timestamp",
-                        "value": "2015-12-23T14:02:22.338776"
-                    },
-                    {
-                        "type": "string",
-                        "name": "trace_id",
-                        "value": "06320327-2c2c-45ae-923a-515de890276a"
-                    }
-                ],
-                "raw": {},
-                "generated": "2015-12-23T10:41:38.415793",
-                "event_type": "profiler.main",
-                "message_id": "65fc1553-3082-4a6f-9d1e-0e3183f57a47"},
-            {
-                "traits":
-                    [
-                        {
-                            "type": "string",
-                            "name": "base_id",
-                            "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
-                        },
-                        {
-                            "type": "string",
-                            "name": "host",
-                            "value": "ubuntu"
-                        },
-                        {
-                            "type": "string",
-                            "name": "name",
-                            "value": "wsgi-stop"
-                        },
-                        {
-                            "type": "string",
-                            "name": "parent_id",
-                            "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
-                        },
-                        {
-                            "type": "string",
-                            "name": "project",
-                            "value": "keystone"
-                        },
-                        {
-                            "type": "string",
-                            "name": "service",
-                            "value": "main"
-                        },
-                        {
-                            "type": "string",
-                            "name": "timestamp",
-                            "value": "2015-12-23T14:02:22.380405"
-                        },
-                        {
-                            "type": "string",
-                            "name": "trace_id",
-                            "value": "016c97fd-87f3-40b2-9b55-e431156b694b"
-                        }
-                    ],
-                "raw": {},
-                "generated": "2015-12-23T10:41:38.406052",
-                "event_type": "profiler.main",
-                "message_id": "3256d9f1-48ba-4ac5-a50b-64fa42c6e264"},
-            {
-                "traits":
-                    [
-                        {
-                            "type": "string",
-                            "name": "base_id",
-                            "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
-                        },
-                        {
-                            "type": "string",
-                            "name": "db.params",
-                            "value": "[]"
-                        },
-                        {
-                            "type": "string",
-                            "name": "db.statement",
-                            "value": "SELECT 1"
-                        },
-                        {
-                            "type": "string",
-                            "name": "host",
-                            "value": "ubuntu"
-                        },
-                        {
-                            "type": "string",
-                            "name": "name",
-                            "value": "db-start"
-                        },
-                        {
-                            "type": "string",
-                            "name": "parent_id",
-                            "value": "06320327-2c2c-45ae-923a-515de890276a"
-                        },
-                        {
-                            "type": "string",
-                            "name": "project",
-                            "value": "keystone"
-                        },
-                        {
-                            "type": "string",
-                            "name": "service",
-                            "value": "main"
-                        },
-                        {
-                            "type": "string",
-                            "name": "timestamp",
-                            "value": "2015-12-23T14:02:22.395365"
-                        },
-                        {
-                            "type": "string",
-                            "name": "trace_id",
-                            "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"
-                        }
-                    ],
-                "raw": {},
-                "generated": "2015-12-23T10:41:38.984161",
-                "event_type": "profiler.main",
-                "message_id": "60368aa4-16f0-4f37-a8fb-89e92fdf36ff"
-            },
-            {
-                "traits":
-                    [
-                        {
-                            "type": "string",
-                            "name": "base_id",
-                            "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
-                        },
-                        {
-                            "type": "string",
-                            "name": "host",
-                            "value": "ubuntu"
-                        },
-                        {
-                            "type": "string",
-                            "name": "name",
-                            "value": "db-stop"
-                        },
-                        {
-                            "type": "string",
-                            "name": "parent_id",
-                            "value": "06320327-2c2c-45ae-923a-515de890276a"
-                        },
-                        {
-                            "type": "string",
-                            "name": "project",
-                            "value": "keystone"
-                        },
-                        {
-                            "type": "string",
-                            "name": "service",
-                            "value": "main"
-                        },
-                        {
-                            "type": "string",
-                            "name": "timestamp",
-                            "value": "2015-12-23T14:02:22.415486"
-                        },
-                        {
-                            "type": "string",
-                            "name": "trace_id",
-                            "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"
-                        }
-                    ],
-                "raw": {},
-                "generated": "2015-12-23T10:41:39.019378",
-                "event_type": "profiler.main",
-                "message_id": "3fbeb339-55c5-4f28-88e4-15bee251dd3d"
-            },
-            {
-                "traits":
-                    [
-                        {
-                            "type": "string",
-                            "name": "base_id",
-                            "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
-                        },
-                        {
-                            "type": "string",
-                            "name": "host",
-                            "value": "ubuntu"
-                        },
-                        {
-                            "type": "string",
-                            "name": "method",
-                            "value": "GET"
-                        },
-                        {
-                            "type": "string",
-                            "name": "name",
-                            "value": "wsgi-start"
-                        },
-                        {
-                            "type": "string",
-                            "name": "parent_id",
-                            "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
-                        },
-                        {
-                            "type": "string",
-                            "name": "project",
-                            "value": "keystone"
-                        },
-                        {
-                            "type": "string",
-                            "name": "service",
-                            "value": "main"
-                        },
-                        {
-                            "type": "string",
-                            "name": "timestamp",
-                            "value": "2015-12-23T14:02:22.427444"
-                        },
-                        {
-                            "type": "string",
-                            "name": "trace_id",
-                            "value": "016c97fd-87f3-40b2-9b55-e431156b694b"
-                        }
-                    ],
-                "raw": {},
-                "generated": "2015-12-23T10:41:38.360409",
-                "event_type": "profiler.main",
-                "message_id": "57b971a9-572f-4f29-9838-3ed2564c6b5b"
-            }
-        ]
-
-        expected = {"children": [
-            {"children": [{"children": [],
-                           "info": {"finished": 76,
-                                    "host": "ubuntu",
-                                    "meta.raw_payload.db-start": {},
-                                    "meta.raw_payload.db-stop": {},
-                                    "name": "db",
-                                    "project": "keystone",
-                                    "service": "main",
-                                    "started": 56},
-                           "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
-                           "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"}
-                          ],
-             "info": {"finished": 0,
-                      "host": "ubuntu",
-                      "meta.raw_payload.wsgi-start": {},
-                      "name": "wsgi",
-                      "project": "keystone",
-                      "service": "main",
-                      "started": 0},
-             "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
-             "trace_id": "06320327-2c2c-45ae-923a-515de890276a"},
-            {"children": [],
-             "info": {"finished": 41,
-                      "host": "ubuntu",
-                      "meta.raw_payload.wsgi-start": {},
-                      "meta.raw_payload.wsgi-stop": {},
-                      "name": "wsgi",
-                      "project": "keystone",
-                      "service": "main",
-                      "started": 88},
-             "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
-             "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}],
-            "info": {"finished": 88, "name": "total", "started": 0}}
-
-        self.assertEqual(expected, ceilometer.parse_notifications(events))
-
-    def test_get_notifications(self):
-        mock_ceil_client = mock.MagicMock()
-        results = [mock.MagicMock(), mock.MagicMock()]
-        mock_ceil_client.events.list.return_value = results
-        base_id = "10"
-
-        result = ceilometer.get_notifications(mock_ceil_client, base_id)
-
-        expected_filter = [{"field": "base_id", "op": "eq", "value": base_id}]
-        mock_ceil_client.events.list.assert_called_once_with(expected_filter,
-                                                             limit=100000)
-        self.assertEqual(result, [results[0].to_dict(), results[1].to_dict()])

test-requirements.txt

@@ -10,4 +10,6 @@ oslosphinx>=2.5.0,!=3.4.0  # Apache-2.0
 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
 
 # Bandit security code scanner
-bandit>=0.17.3 # Apache-2.0
+bandit>=0.17.3 # Apache-2.0
+
+python-ceilometerclient>=2.2.1 # Apache-2.0