Add support for Swift incoming/outgoing trafic metering

This adds a middleware for Swift that meters incoming and outgoing bytes.

This is part of blueprint pollster-swift.

Change-Id: I94f330ee4cf5df8a743c77fcfae9efd505568060
Signed-off-by: Julien Danjou <julien@danjou.info>

7 files changed, 268 insertions, 8 deletions

diff --git a/ceilometer/objectstore/swift_middleware.py b/ceilometer/objectstore/swift_middleware.py
new file mode 100644
index 00000000..18e3afdb
--- /dev/null
+++ b/ceilometer/objectstore/swift_middleware.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2012 eNovance <licensing@enovance.com>
+#
+# Author: Julien Danjou <julien@danjou.info>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from ceilometer import publish
+from ceilometer import counter
+from ceilometer.openstack.common import cfg
+from ceilometer.openstack.common import context
+from ceilometer.openstack.common import timeutils
+
+from swift.common.swob import Request
+from swift.common.utils import split_path
+try:
+    # Swift > 1.7.5
+    from swift.common.utils import InputProxy
+except ImportError:
+    # Swift <= 1.7.5
+    from swift.common.middleware.proxy_logging import InputProxy
+
+
+class CeilometerMiddleware(object):
+    """
+    Ceilometer middleware used for counting requests.
+    """
+
+    def __init__(self, app, conf):
+        self.app = app
+        cfg.CONF([], project='ceilometer')
+
+    def __call__(self, env, start_response):
+        start_response_args = [None]
+        input_proxy = InputProxy(env['wsgi.input'])
+        env['wsgi.input'] = input_proxy
+
+        def my_start_response(status, headers, exc_info=None):
+            start_response_args[0] = (status, list(headers), exc_info)
+
+        def iter_response(iterable):
+            if start_response_args[0]:
+                start_response(*start_response_args[0])
+            bytes_sent = 0
+            try:
+                for chunk in iterable:
+                    if chunk:
+                        bytes_sent += len(chunk)
+                    yield chunk
+            finally:
+                self.publish_counter(env,
+                                     input_proxy.bytes_received,
+                                     bytes_sent)
+
+        try:
+            iterable = self.app(env, my_start_response)
+        except Exception:
+            self.publish_counter(env, input_proxy.bytes_received, 0)
+            raise
+        else:
+            return iter_response(iterable)
+
+    @staticmethod
+    def publish_counter(env, bytes_received, bytes_sent):
+        req = Request(env)
+        version, account, container, obj = split_path(req.path, 1, 4, True)
+        now = timeutils.utcnow().isoformat()
+
+        if bytes_received:
+            publish.publish_counter(context.get_admin_context(),
+                                    counter.Counter(
+                                        name='storage.objects.incoming.bytes',
+                                        type='delta',
+                                        volume=bytes_received,
+                                        user_id=env.get('HTTP_X_USER_ID'),
+                                        project_id=env.get('HTTP_X_TENANT_ID'),
+                                        resource_id=account.partition(
+                                            'AUTH_')[2],
+                                        timestamp=now,
+                                        resource_metadata={
+                                            "path": req.path,
+                                            "version": version,
+                                            "container": container,
+                                            "object": obj,
+                                    }),
+                                    cfg.CONF.metering_topic,
+                                    cfg.CONF.metering_secret,
+                                    cfg.CONF.counter_source)
+
+        if bytes_sent:
+            publish.publish_counter(context.get_admin_context(),
+                                    counter.Counter(
+                                        name='storage.objects.outgoing.bytes',
+                                        type='delta',
+                                        volume=bytes_sent,
+                                        user_id=env.get('HTTP_X_USER_ID'),
+                                        project_id=env.get('HTTP_X_TENANT_ID'),
+                                        resource_id=account.partition(
+                                            'AUTH_')[2],
+                                        timestamp=now,
+                                        resource_metadata={
+                                            "path": req.path,
+                                            "version": version,
+                                            "container": container,
+                                            "object": obj,
+                                    }),
+                                    cfg.CONF.metering_topic,
+                                    cfg.CONF.metering_secret,
+                                    cfg.CONF.counter_source)
+
+
+def filter_factory(global_conf, **local_conf):
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def ceilometer_filter(app):
+        return CeilometerMiddleware(app, conf)
+    return ceilometer_filter
diff --git a/doc/source/install.rst b/doc/source/install.rst
index 77d6e5cf..ee0306eb 100644
--- a/doc/source/install.rst
+++ b/doc/source/install.rst
@@ -98,6 +98,15 @@ Installing the Collector
                             --user_id $CEILOMETER_USER \
                             --role_id 462fa46c13fd4798a95a3bfbe27b5e54
 
+   You'll also need to add the Ceilometer middleware to Swift to account for
+   incoming and outgoing traffic, adding this lines to
+   ``/etc/swift/proxy-server.conf``::
+
+     [filter:ceilometer]
+     use = egg:ceilometer#swift
+
+   And adding ``ceilometer`` in the ``pipeline`` of that same file.
+
 4. Install MongoDB.
 
    Follow the instructions to install the MongoDB_ package for your
diff --git a/doc/source/measurements.rst b/doc/source/measurements.rst
index 445ec6a4..7b71d4e8 100644
--- a/doc/source/measurements.rst
+++ b/doc/source/measurements.rst
@@ -109,13 +109,15 @@ volume.size               Gauge            GB  vol ID    Size of volume
 Object Storage (Swift)
 ======================
 
-==========================  ==========  ==========  ========  ==================================================
-Name                        Type        Volume      Resource  Note
-==========================  ==========  ==========  ========  ==================================================
-storage.objects             Gauge          objects  store ID  Number of objects
-storage.objects.size        Gauge            bytes  store ID  Total size of stored objects
-storage.objects.containers  Gauge       containers  store ID  Number of containers
-==========================  ==========  ==========  ========  ==================================================
+==========================      ==========  ==========  ========  ==================================================
+Name                            Type        Volume      Resource  Note
+==========================      ==========  ==========  ========  ==================================================
+storage.objects                 Gauge          objects  store ID  Number of objects
+storage.objects.size            Gauge            bytes  store ID  Total size of stored objects
+storage.objects.containers      Gauge       containers  store ID  Number of containers
+storage.objects.incoming.bytes  Delta            bytes  store ID  Number of incoming bytes
+storage.objects.outgoing.bytes  Delta            bytes  store ID  Number of outgoing bytes
+==============================  ==========  ==========  ========  ==================================================
 
 Dynamically retrieving the Meters via ceilometer client
 =======================================================
diff --git a/setup.py b/setup.py
index fa345f82..7a79cb96 100755
--- a/setup.py
+++ b/setup.py
@@ -132,5 +132,8 @@ setuptools.setup(
 
     [ceilometer.compute.virt]
     libvirt = ceilometer.compute.virt.libvirt.inspector:LibvirtInspector
+
+    [paste.filter_factory]
+    swift=ceilometer.objectstore.swift_middleware:filter_factory
     """),
     )
diff --git a/tests/objectstore/test_swift_middleware.py b/tests/objectstore/test_swift_middleware.py
new file mode 100644
index 00000000..7436e417
--- /dev/null
+++ b/tests/objectstore/test_swift_middleware.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2012 eNovance <licensing@enovance.com>
+#
+# Author: Julien Danjou <julien@danjou.info>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import cStringIO as StringIO
+from webob import Request
+
+from ceilometer.tests import base
+from ceilometer.objectstore import swift_middleware
+from ceilometer.openstack.common import rpc
+
+
+class FakeApp(object):
+    def __init__(self, body=['This string is 28 bytes long']):
+        self.body = body
+
+    def __call__(self, env, start_response):
+        start_response('200 OK', [('Content-Type', 'text/plain'),
+                            ('Content-Length', str(sum(map(len, self.body))))])
+        while env['wsgi.input'].read(5):
+            pass
+        return self.body
+
+
+class TestSwiftMiddleware(base.TestCase):
+
+    def setUp(self):
+        super(TestSwiftMiddleware, self).setUp()
+        self.notifications = []
+        self.stubs.Set(rpc, 'cast', self._faux_notify)
+
+    @staticmethod
+    def start_response(*args):
+            pass
+
+    def _faux_notify(self, context, topic, msg):
+        self.notifications.append((topic, msg))
+
+    def test_get(self):
+        app = swift_middleware.CeilometerMiddleware(FakeApp(), {})
+        req = Request.blank('/1.0/account/container/obj',
+                            environ={'REQUEST_METHOD': 'GET'})
+        resp = app(req.environ, self.start_response)
+        self.assertEqual(list(resp), ["This string is 28 bytes long"])
+        self.assertEqual(len(self.notifications), 2)
+        data = self.notifications[0][1]['args']['data']
+        self.assertEqual(data['counter_volume'], 28)
+        self.assertEqual(data['resource_metadata']['version'], '1.0')
+        self.assertEqual(data['resource_metadata']['container'], 'container')
+        self.assertEqual(data['resource_metadata']['object'], 'obj')
+
+    def test_put(self):
+        app = swift_middleware.CeilometerMiddleware(FakeApp(body=['']), {})
+        req = Request.blank('/1.0/account/container/obj',
+                            environ={'REQUEST_METHOD': 'GET',
+                                     'wsgi.input':
+                                     StringIO.StringIO('some stuff')})
+        resp = list(app(req.environ, self.start_response))
+        self.assertEqual(len(self.notifications), 2)
+        data = self.notifications[0][1]['args']['data']
+        self.assertEqual(data['counter_volume'], 10)
+        self.assertEqual(data['resource_metadata']['version'], '1.0')
+        self.assertEqual(data['resource_metadata']['container'], 'container')
+        self.assertEqual(data['resource_metadata']['object'], 'obj')
+
+    def test_post(self):
+        app = swift_middleware.CeilometerMiddleware(FakeApp(body=['']), {})
+        req = Request.blank('/1.0/account/container/obj',
+                            environ={'REQUEST_METHOD': 'POST',
+                                     'wsgi.input':
+                                     StringIO.StringIO('some other stuff')})
+        resp = list(app(req.environ, self.start_response))
+        self.assertEqual(len(self.notifications), 2)
+        data = self.notifications[0][1]['args']['data']
+        self.assertEqual(data['counter_volume'], 16)
+        self.assertEqual(data['resource_metadata']['version'], '1.0')
+        self.assertEqual(data['resource_metadata']['container'], 'container')
+        self.assertEqual(data['resource_metadata']['object'], 'obj')
+
+    def test_get_container(self):
+        app = swift_middleware.CeilometerMiddleware(FakeApp(), {})
+        req = Request.blank('/1.0/account/container',
+                            environ={'REQUEST_METHOD': 'GET'})
+        resp = list(app(req.environ, self.start_response))
+        self.assertEqual(len(self.notifications), 2)
+        data = self.notifications[0][1]['args']['data']
+        self.assertEqual(data['counter_volume'], 28)
+        self.assertEqual(data['resource_metadata']['version'], '1.0')
+        self.assertEqual(data['resource_metadata']['container'], 'container')
+        self.assertEqual(data['resource_metadata']['object'], None)
diff --git a/tools/test-requires b/tools/test-requires
index 2935d237..ad7ffd13 100644
--- a/tools/test-requires
+++ b/tools/test-requires
@@ -22,3 +22,7 @@ setuptools-git>=0.4
 # very soon.
 hg+https://bitbucket.org/cdevienne/wsme
 pecan
+# We should use swift>1.7.5, but it's not yet available
+swift
+# Swift dep that is not necessary if we depend on swift>1.7.5
+netifaces
diff --git a/tools/test-requires-folsom b/tools/test-requires-folsom
index 202e2472..82dd0531 100644
--- a/tools/test-requires-folsom
+++ b/tools/test-requires-folsom
@@ -20,4 +20,8 @@ setuptools-git>=0.4
 # checkout on bitbucket. I hope to have that resolved
 # very soon.
 hg+https://bitbucket.org/cdevienne/wsme
-pecan
\ No newline at end of file
+pecan
+# We should use swift>1.7.5, but it's not yet available
+swift
+# Swift dep that is not necessary if we depend on swift>1.7.5
+netifaces