Add new middleware to send API data to statsd

The new StatsMiddleware is a Paste filter that examines the URL path and
request method, and sends a stat count and a timer to a statsd host
whose name is based on the path/method.

If your statsd is configured to send stats to Graphite, you'll end up
with stat names of the form:
timer.<appname>.<METHOD>.<path>.<from>.<url>

Because a dot has special meaning in Graphite, dots in API versions that
appear in the path will be replaced with _, so for example v2.1 becomes v2_1,
and v1.0 becomes v1_0.

Change-Id: Ieaffeded1bf81c0782d88f49b6f5209f11744899

3 files changed, 289 insertions, 0 deletions

diff --git a/oslo_middleware/stats.py b/oslo_middleware/stats.py
new file mode 100644
index 0000000..9839fd5
--- /dev/null
+++ b/oslo_middleware/stats.py
@@ -0,0 +1,131 @@
+# Copyright (c) 2016 Cisco Systems
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import re
+
+import statsd
+import webob.dec
+
+from oslo_middleware import base
+
+LOG = logging.getLogger(__name__)
+VERSION_REGEX = re.compile("/(v[0-9]{1}\.[0-9]{1})")
+UUID_REGEX = re.compile(
+    '.*(\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}).*a',
+    re.IGNORECASE)
+# UUIDs without the - char, used in some places in Nova URLs.
+SHORT_UUID_REGEX = re.compile('.*(\.[0-9a-fA-F]{32}).*')
+
+
+class StatsMiddleware(base.ConfigurableMiddleware):
+    """Send stats to statsd based on API requests.
+
+    Examines the URL path and request method, and sends a stat count and timer
+    to a statsd host based on the path/method.
+
+    If your statsd is configured to send stats to Graphite, you'll end up with
+    stat names of the form::
+
+        timer.<appname>.<METHOD>.<path>.<from>.<url>
+
+    Note that URLs with versions in them (pretty much all of Openstack)
+    are always processed to replace the dot with _, so for example v2.0
+    becomes v2_0, and v1.1 becomes v1_1, since a dot '.' has special
+    meaning in Graphite.
+
+    The original StatsD is written in nodejs. If you want a Python
+    implementation, install Bucky instead as it's a drop-in replacement
+    (and much nicer IMO).
+
+    The Paste config must contain some parameters. Configure a filter like
+    this::
+
+        [filter:stats]
+        paste.filter_factory = oslo_middleware.stats:StatsMiddleware.factory
+        name = my_application_name  # e.g. 'glance'
+        stats_host = my_statsd_host.example.com
+        # Optional args to further process the stat name that's generated:
+        remove_uuid = True
+        remove_short_uuid = True
+        # The above uuid processing is required in, e.g. Nova, if you want to
+        # collect generic stats rather than one per server instance.
+    """
+
+    def __init__(self, application, conf):
+        super(StatsMiddleware, self).__init__(application, conf)
+        self.application = application
+        self.stat_name = conf.get('name')
+        if self.stat_name is None:
+            raise AttributeError('name must be specified')
+        self.stats_host = conf.get('stats_host')
+        if self.stats_host is None:
+            raise AttributeError('stats_host must be specified')
+        self.remove_uuid = conf.get('remove_uuid', False)
+        self.remove_short_uuid = conf.get('remove_short_uuid', False)
+        self.statsd = statsd.StatsClient(self.stats_host)
+
+    @staticmethod
+    def strip_short_uuid(path):
+        """Remove short-form UUID from supplied path.
+
+        Only call after replacing slashes with dots in path.
+        """
+        match = SHORT_UUID_REGEX.match(path)
+        if match is None:
+            return path
+        return path.replace(match.group(1), '')
+
+    @staticmethod
+    def strip_uuid(path):
+        """Remove normal-form UUID from supplied path.
+
+        Only call after replacing slashes with dots in path.
+        """
+        match = UUID_REGEX.match(path)
+        if match is None:
+            return path
+        return path.replace(match.group(1), '')
+
+    @staticmethod
+    def strip_dot_from_version(path):
+        # Replace vN.N with vNN.
+        match = VERSION_REGEX.match(path)
+        if match is None:
+            return path
+        return path.replace(match.group(1), match.group(1).replace('.', ''))
+
+    @webob.dec.wsgify
+    def __call__(self, request):
+        path = request.path
+        path = self.strip_dot_from_version(path)
+
+        # Remove leading slash, if any, so we can be sure of the number
+        # of dots just below.
+        path = path.lstrip('/')
+
+        stat = "{name}.{method}".format(
+            name=self.stat_name, method=request.method)
+        if path != '':
+            stat += '.' + path.replace('/', '.')
+
+        if self.remove_short_uuid:
+            stat = self.strip_short_uuid(stat)
+
+        if self.remove_uuid:
+            stat = self.strip_uuid(stat)
+
+        LOG.debug("Incrementing stat count %s", stat)
+        with self.statsd.timer(stat):
+            return request.get_response(self.application)
diff --git a/oslo_middleware/tests/test_stats.py b/oslo_middleware/tests/test_stats.py
new file mode 100644
index 0000000..1e2e412
--- /dev/null
+++ b/oslo_middleware/tests/test_stats.py
@@ -0,0 +1,157 @@
+# Copyright (c) 2016 Cisco Systems
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import uuid
+
+import mock
+from oslotest import base as test_base
+import statsd
+import webob.dec
+import webob.exc
+
+from oslo_middleware import stats
+
+
+class TestStaticMethods(test_base.BaseTestCase):
+
+    def test_removes_uuid(self):
+        # Generate a long-format UUID (standard form).
+        id = str(uuid.uuid4())
+        path = "foo.{uuid}.bar".format(uuid=id)
+        stat = stats.StatsMiddleware.strip_uuid(path)
+        self.assertEqual("foo.bar", stat)
+
+    def test_removes_short_uuid(self):
+        id = uuid.uuid4().hex
+        path = "foo.{uuid}.bar".format(uuid=id)
+        stat = stats.StatsMiddleware.strip_short_uuid(path)
+        self.assertEqual("foo.bar", stat)
+
+    def test_strips_dots_from_version(self):
+        # NOTE(bigjools): Good testing practice says to randomise inputs
+        # that have no meaning to the test. However my reviewer has said
+        # not to do this, so the versions are static.
+        path = "/v1.2/foo.bar/bar.foo"
+        stat = stats.StatsMiddleware.strip_dot_from_version(path)
+        self.assertEqual("/v12/foo.bar/bar.foo", stat)
+
+
+class TestStatsMiddleware(test_base.BaseTestCase):
+
+    def setUp(self):
+        super(TestStatsMiddleware, self).setUp()
+        self.patch(statsd, 'StatsClient', mock.MagicMock())
+
+    def make_stats_middleware(self, stat_name=None, stats_host=None,
+                              remove_uuid=False, remove_short_uuid=False):
+        if stat_name is None:
+            stat_name = uuid.uuid4().hex
+        if stats_host is None:
+            stats_host = uuid.uuid4().hex
+
+        conf = dict(
+            name=stat_name,
+            stats_host=stats_host,
+            remove_uuid=remove_uuid,
+            remove_short_uuid=remove_short_uuid,
+        )
+
+        @webob.dec.wsgify
+        def fake_application(req):
+            return 'Hello, World'
+
+        return stats.StatsMiddleware(fake_application, conf)
+
+    def get_random_method(self):
+        # NOTE(bigjools): Good testing practice says to randomise inputs
+        # that have no meaning to the test. However my reviewer has said
+        # not to do this, so the methods are static.
+        return "methodXVNMapyr"
+
+    def perform_request(self, app, path, method):
+        req = webob.Request.blank(path, method=method)
+        return req.get_response(app)
+
+    def test_sends_counter_to_statsd(self):
+        app = self.make_stats_middleware()
+        random_method = self.get_random_method()
+        path = '/test/foo/bar'
+
+        self.perform_request(app, path, random_method)
+
+        expected_stat = "{name}.{method}.{path}".format(
+            name=app.stat_name, method=random_method,
+            path=path.lstrip('/').replace('/', '.'))
+        app.statsd.timer.assert_called_once_with(expected_stat)
+
+    def test_strips_uuid_if_configured(self):
+        app = self.make_stats_middleware(remove_uuid=True)
+        random_method = self.get_random_method()
+        random_uuid = str(uuid.uuid4())
+        path = '/foo/{uuid}/bar'.format(uuid=random_uuid)
+
+        self.perform_request(app, path, random_method)
+
+        expected_stat = "{name}.{method}.foo.bar".format(
+            name=app.stat_name, method=random_method)
+        app.statsd.timer.assert_called_once_with(expected_stat)
+
+    def test_strips_short_uuid_if_configured(self):
+        app = self.make_stats_middleware(remove_short_uuid=True)
+        random_method = self.get_random_method()
+        random_uuid = uuid.uuid4().hex
+        path = '/foo/{uuid}/bar'.format(uuid=random_uuid)
+
+        self.perform_request(app, path, random_method)
+
+        expected_stat = "{name}.{method}.foo.bar".format(
+            name=app.stat_name, method=random_method)
+        app.statsd.timer.assert_called_once_with(expected_stat)
+
+    def test_strips_both_uuid_types_if_configured(self):
+        app = self.make_stats_middleware(
+            remove_uuid=True, remove_short_uuid=True)
+        random_method = self.get_random_method()
+        random_short_uuid = uuid.uuid4().hex
+        random_uuid = str(uuid.uuid4())
+        path = '/foo/{uuid}/bar/{short_uuid}'.format(
+            uuid=random_uuid, short_uuid=random_short_uuid)
+
+        self.perform_request(app, path, random_method)
+
+        expected_stat = "{name}.{method}.foo.bar".format(
+            name=app.stat_name, method=random_method)
+        app.statsd.timer.assert_called_once_with(expected_stat)
+
+    def test_always_mutates_version_id(self):
+        app = self.make_stats_middleware()
+        random_method = self.get_random_method()
+        path = '/v2.1/foo/bar'
+
+        self.perform_request(app, path, random_method)
+
+        expected_stat = "{name}.{method}.v21.foo.bar".format(
+            name=app.stat_name, method=random_method)
+        app.statsd.timer.assert_called_once_with(expected_stat)
+
+    def test_empty_path_has_sane_stat_name(self):
+        app = self.make_stats_middleware()
+        random_method = self.get_random_method()
+        path = '/'
+
+        self.perform_request(app, path, random_method)
+
+        expected_stat = "{name}.{method}".format(
+            name=app.stat_name, method=random_method)
+        app.statsd.timer.assert_called_once_with(expected_stat)
diff --git a/requirements.txt b/requirements.txt
index 381e433..384efd3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,3 +12,4 @@ six>=1.9.0 # MIT
 stevedore>=1.16.0 # Apache-2.0
 WebOb>=1.2.3 # MIT
 debtcollector>=1.2.0 # Apache-2.0
+statsd>=3.2.1 # MIT