diff options
| -rw-r--r-- | pycadf/audit/api.py | 32 | ||||
| -rw-r--r-- | pycadf/middleware/__init__.py | 0 | ||||
| -rw-r--r-- | pycadf/middleware/audit.py | 45 | ||||
| -rw-r--r-- | pycadf/middleware/base.py | 56 | ||||
| -rw-r--r-- | pycadf/middleware/notifier.py | 140 | ||||
| -rw-r--r-- | pycadf/tests/audit/test_api.py | 31 | ||||
| -rw-r--r-- | pycadf/tests/base.py | 8 | ||||
| -rw-r--r-- | pycadf/tests/middleware/__init__.py | 0 | ||||
| -rw-r--r-- | pycadf/tests/middleware/test_audit.py | 159 | ||||
| -rw-r--r-- | requirements.txt | 3 | ||||
| -rw-r--r-- | setup.cfg | 1 | ||||
| -rw-r--r-- | test-requirements.txt | 2 |
12 files changed, 443 insertions, 34 deletions
diff --git a/pycadf/audit/api.py b/pycadf/audit/api.py index 3b7686a..31ec712 100644 --- a/pycadf/audit/api.py +++ b/pycadf/audit/api.py @@ -18,9 +18,9 @@ import ast import collections import os -from oslo.config import cfg import re +from oslo.config import cfg from six.moves import configparser from six.moves.urllib import parse as urlparse @@ -37,16 +37,14 @@ from pycadf import resource from pycadf import tag from pycadf import timestamp +#NOTE(gordc): remove cfg once we move over to this middleware version CONF = cfg.CONF -opts = [ - cfg.StrOpt('api_audit_map', - default='api_audit_map.conf', - help='File containing mapping for api paths and ' - 'service endpoints'), -] +opts = [cfg.StrOpt('api_audit_map', + default='api_audit_map.conf', + help='File containing mapping for api paths and ' + 'service endpoints')] CONF.register_opts(opts, group='audit') - AuditMap = collections.namedtuple('AuditMap', ['path_kw', 'custom_actions', @@ -54,7 +52,7 @@ AuditMap = collections.namedtuple('AuditMap', 'default_target_endpoint_type']) -def _configure_audit_map(): +def _configure_audit_map(cfg_file): """Configure to recognize and map known api paths.""" path_kw = {} @@ -62,10 +60,6 @@ def _configure_audit_map(): service_endpoints = {} default_target_endpoint_type = None - cfg_file = CONF.audit.api_audit_map - if not os.path.exists(CONF.audit.api_audit_map): - cfg_file = cfg.CONF.find_file(CONF.audit.api_audit_map) - if cfg_file: try: map_conf = configparser.SafeConfigParser() @@ -119,12 +113,17 @@ class PycadfAuditApiConfigError(Exception): class OpenStackAuditApi(object): - _MAP = None - Service = collections.namedtuple('Service', ['id', 'name', 'type', 'admin_endp', 'public_endp', 'private_endp']) + def __init__(self, map_file=None): + if map_file is None: + map_file = CONF.audit.api_audit_map + if not os.path.exists(CONF.audit.api_audit_map): + map_file = cfg.CONF.find_file(CONF.audit.api_audit_map) + self._MAP = _configure_audit_map(map_file) + def _get_action(self, req): """Take a given Request, parse url path to calculate action type. @@ -207,9 +206,6 @@ class OpenStackAuditApi(object): return service_type + type_uri def create_event(self, req, correlation_id): - if not self._MAP: - self._MAP = _configure_audit_map() - action = self._get_action(req) initiator_host = host.Host(address=req.client_addr, agent=req.user_agent) diff --git a/pycadf/middleware/__init__.py b/pycadf/middleware/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pycadf/middleware/__init__.py diff --git a/pycadf/middleware/audit.py b/pycadf/middleware/audit.py new file mode 100644 index 0000000..1293cb6 --- /dev/null +++ b/pycadf/middleware/audit.py @@ -0,0 +1,45 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Attach open standard audit information to request.environ + +AuditMiddleware filter should be place after Keystone's auth_token middleware +in the pipeline so that it can utilise the information Keystone provides. + +""" +from pycadf.audit import api as cadf_api + +from pycadf.middleware import notifier + + +class AuditMiddleware(notifier.RequestNotifier): + + def __init__(self, app, **conf): + super(AuditMiddleware, self).__init__(app, **conf) + map_file = conf.get('audit_map_file', None) + self.cadf_audit = cadf_api.OpenStackAuditApi(map_file) + + @notifier.log_and_ignore_error + def process_request(self, request): + self.cadf_audit.append_audit_event(request) + super(AuditMiddleware, self).process_request(request) + + @notifier.log_and_ignore_error + def process_response(self, request, response, + exception=None, traceback=None): + self.cadf_audit.mod_audit_event(request, response) + super(AuditMiddleware, self).process_response(request, response, + exception, traceback) diff --git a/pycadf/middleware/base.py b/pycadf/middleware/base.py new file mode 100644 index 0000000..464a1cc --- /dev/null +++ b/pycadf/middleware/base.py @@ -0,0 +1,56 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Base class(es) for WSGI Middleware.""" + +import webob.dec + + +class Middleware(object): + """Base WSGI middleware wrapper. + + These classes require an application to be initialized that will be called + next. By default the middleware will simply call its wrapped app, or you + can override __call__ to customize its behavior. + """ + + @classmethod + def factory(cls, global_conf, **local_conf): + """Factory method for paste.deploy.""" + return cls + + def __init__(self, application): + self.application = application + + def process_request(self, req): + """Called on each request. + + If this returns None, the next application down the stack will be + executed. If it returns a response then that response will be returned + and execution will stop here. + """ + return None + + def process_response(self, response): + """Do whatever you'd like to the response.""" + return response + + @webob.dec.wsgify + def __call__(self, req): + response = self.process_request(req) + if response: + return response + response = req.get_response(self.application) + return self.process_response(response) diff --git a/pycadf/middleware/notifier.py b/pycadf/middleware/notifier.py new file mode 100644 index 0000000..fc921f8 --- /dev/null +++ b/pycadf/middleware/notifier.py @@ -0,0 +1,140 @@ +# Copyright (c) 2013 eNovance +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Send notifications on request +""" +import os.path +import sys +import traceback as tb + +from oslo.config import cfg +import oslo.messaging +import six +import webob.dec + +from pycadf.middleware import base +from pycadf.openstack.common import context +from pycadf.openstack.common.gettextutils import _ # noqa + +LOG = None + + +def log_and_ignore_error(fn): + def wrapped(*args, **kwargs): + try: + return fn(*args, **kwargs) + except Exception as e: + if LOG: + LOG.exception(_('An exception occurred processing ' + 'the API call: %s ') % e) + return wrapped + + +class RequestNotifier(base.Middleware): + """Send notification on request.""" + + @classmethod + def factory(cls, global_conf, **local_conf): + """Factory method for paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def _factory(app): + return cls(app, **conf) + return _factory + + def __init__(self, app, **conf): + global LOG + + proj = cfg.CONF.project + TRANSPORT_ALIASES = {} + if proj: + log_mod = '%s.openstack.common.log' % proj + if log_mod in sys.modules: + LOG = sys.modules[log_mod].getLogger(__name__) + # Aliases to support backward compatibility + TRANSPORT_ALIASES = { + '%s.openstack.common.rpc.impl_kombu' % proj: 'rabbit', + '%s.openstack.common.rpc.impl_qpid' % proj: 'qpid', + '%s.openstack.common.rpc.impl_zmq' % proj: 'zmq', + '%s.rpc.impl_kombu' % proj: 'rabbit', + '%s.rpc.impl_qpid' % proj: 'qpid', + '%s.rpc.impl_zmq' % proj: 'zmq', + } + + self.service_name = conf.get('service_name') + self.ignore_req_list = [x.upper().strip() for x in + conf.get('ignore_req_list', '').split(',')] + self.notifier = oslo.messaging.Notifier( + oslo.messaging.get_transport(cfg.CONF, aliases=TRANSPORT_ALIASES), + os.path.basename(sys.argv[0])) + super(RequestNotifier, self).__init__(app) + + @staticmethod + def environ_to_dict(environ): + """Following PEP 333, server variables are lower case, so don't + include them. + """ + return dict((k, v) for k, v in six.iteritems(environ) + if k.isupper()) + + @log_and_ignore_error + def process_request(self, request): + request.environ['HTTP_X_SERVICE_NAME'] = \ + self.service_name or request.host + payload = { + 'request': self.environ_to_dict(request.environ), + } + + self.notifier.info(context.get_admin_context().to_dict(), + 'http.request', payload) + + @log_and_ignore_error + def process_response(self, request, response, + exception=None, traceback=None): + payload = { + 'request': self.environ_to_dict(request.environ), + } + + if response: + payload['response'] = { + 'status': response.status, + 'headers': response.headers, + } + + if exception: + payload['exception'] = { + 'value': repr(exception), + 'traceback': tb.format_tb(traceback) + } + + self.notifier.info(context.get_admin_context().to_dict(), + 'http.response', payload) + + @webob.dec.wsgify + def __call__(self, req): + if req.method in self.ignore_req_list: + return req.get_response(self.application) + else: + self.process_request(req) + try: + response = req.get_response(self.application) + except Exception: + exc_type, value, traceback = sys.exc_info() + self.process_response(req, None, value, traceback) + raise + else: + self.process_response(req, response) + return response diff --git a/pycadf/tests/audit/test_api.py b/pycadf/tests/audit/test_api.py index 10abe1c..c70086e 100644 --- a/pycadf/tests/audit/test_api.py +++ b/pycadf/tests/audit/test_api.py @@ -14,8 +14,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -from oslo.config import cfg import uuid + +from oslo.config import cfg import webob from pycadf.audit import api @@ -43,14 +44,8 @@ class TestAuditApi(base.TestCase): def setUp(self): super(TestAuditApi, self).setUp() - # set nova CONF.host value - # Set a default location for the api_audit_map config file - cfg.CONF.set_override( - 'api_audit_map', - self.path_get('etc/pycadf/api_audit_map.conf'), - group='audit' - ) - self.audit_api = api.OpenStackAuditApi() + self.audit_api = api.OpenStackAuditApi( + 'etc/pycadf/api_audit_map.conf') def api_request(self, method, url): self.ENV_HEADERS['REQUEST_METHOD'] = method @@ -60,6 +55,18 @@ class TestAuditApi(base.TestCase): self.assertIn('CADF_EVENT_CORRELATION_ID', req.environ) return req + def test_get_list_with_cfg(self): + cfg.CONF.set_override( + 'api_audit_map', + self.path_get('etc/pycadf/api_audit_map.conf'), + group='audit') + self.audit_api = api.OpenStackAuditApi() + req = self.api_request('GET', + 'http://admin_host:8774/v2/' + + str(uuid.uuid4()) + '/servers/') + payload = req.environ['CADF_EVENT'] + self.assertEqual(payload['action'], 'read/list') + def test_get_list(self): req = self.api_request('GET', 'http://admin_host:8774/v2/' + str(uuid.uuid4()) + '/servers') @@ -125,8 +132,7 @@ class TestAuditApi(base.TestCase): f.write("servers = server\n\n") f.write("[service_endpoints]\n") f.write("compute = service/compute") - cfg.CONF.set_override('api_audit_map', tmpfile, group='audit') - self.audit_api = api.OpenStackAuditApi() + self.audit_api = api.OpenStackAuditApi(tmpfile) req = self.api_request('GET', 'http://unknown:8774/v2/' @@ -283,5 +289,4 @@ class TestAuditApiConf(base.TestCase): f.write("api_paths = servers\n\n") f.write("[service_endpoints]\n") f.write("compute = service/compute") - cfg.CONF.set_override('api_audit_map', tmpfile, group='audit') - self.audit_api = api.OpenStackAuditApi() + self.audit_api = api.OpenStackAuditApi(tmpfile) diff --git a/pycadf/tests/base.py b/pycadf/tests/base.py index a231b41..f14f9da 100644 --- a/pycadf/tests/base.py +++ b/pycadf/tests/base.py @@ -18,15 +18,21 @@ """ import fixtures import os.path -from oslo.config import cfg import testtools +from oslo.config import cfg + +from pycadf.openstack.common.fixture import moxstubout + class TestCase(testtools.TestCase): def setUp(self): super(TestCase, self).setUp() self.tempdir = self.useFixture(fixtures.TempDir()) + moxfixture = self.useFixture(moxstubout.MoxStubout()) + self.mox = moxfixture.mox + self.stubs = moxfixture.stubs cfg.CONF([], project='pycadf') def path_get(self, project_file=None): diff --git a/pycadf/tests/middleware/__init__.py b/pycadf/tests/middleware/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pycadf/tests/middleware/__init__.py diff --git a/pycadf/tests/middleware/test_audit.py b/pycadf/tests/middleware/test_audit.py new file mode 100644 index 0000000..86ac5fc --- /dev/null +++ b/pycadf/tests/middleware/test_audit.py @@ -0,0 +1,159 @@ +# Copyright (c) 2014 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import webob + +from pycadf.audit import api as cadf_api +from pycadf.middleware import audit +from pycadf.tests import base + + +class FakeApp(object): + def __call__(self, env, start_response): + body = 'Some response' + start_response('200 OK', [ + ('Content-Type', 'text/plain'), + ('Content-Length', str(sum(map(len, body)))) + ]) + return [body] + + +class FakeFailingApp(object): + def __call__(self, env, start_response): + raise Exception("It happens!") + + +@mock.patch('oslo.messaging.get_transport', mock.MagicMock()) +class AuditMiddlewareTest(base.TestCase): + ENV_HEADERS = {'HTTP_X_SERVICE_CATALOG': + '''[{"endpoints_links": [], + "endpoints": [{"adminURL": + "http://host:8774/v2/admin", + "region": "RegionOne", + "publicURL": + "http://host:8774/v2/public", + "internalURL": + "http://host:8774/v2/internal", + "id": "resource_id"}], + "type": "compute", + "name": "nova"},]''', + 'HTTP_X_USER_ID': 'user_id', + 'HTTP_X_USER_NAME': 'user_name', + 'HTTP_X_AUTH_TOKEN': 'token', + 'HTTP_X_PROJECT_ID': 'tenant_id', + 'HTTP_X_IDENTITY_STATUS': 'Confirmed'} + + def setUp(self): + super(AuditMiddlewareTest, self).setUp() + self.map_file = 'etc/pycadf/api_audit_map.conf' + + def test_api_request(self): + middleware = audit.AuditMiddleware(FakeApp(), + audit_map_file= + 'etc/pycadf/api_audit_map.conf', + service_name='pycadf') + self.ENV_HEADERS['REQUEST_METHOD'] = 'GET' + req = webob.Request.blank('/foo/bar', + environ=self.ENV_HEADERS) + with mock.patch('oslo.messaging.Notifier.info') as notify: + middleware(req) + # Check first notification with only 'request' + call_args = notify.call_args_list[0][0] + self.assertEqual(call_args[1], 'http.request') + self.assertEqual(set(call_args[2].keys()), + set(['request'])) + + request = call_args[2]['request'] + self.assertEqual(request['PATH_INFO'], '/foo/bar') + self.assertEqual(request['REQUEST_METHOD'], 'GET') + self.assertIn('CADF_EVENT', request) + self.assertEqual(request['CADF_EVENT']['outcome'], 'pending') + + # Check second notification with request + response + call_args = notify.call_args_list[1][0] + self.assertEqual(call_args[1], 'http.response') + self.assertEqual(set(call_args[2].keys()), + set(['request', 'response'])) + + request = call_args[2]['request'] + self.assertEqual(request['PATH_INFO'], '/foo/bar') + self.assertEqual(request['REQUEST_METHOD'], 'GET') + self.assertIn('CADF_EVENT', request) + self.assertEqual(request['CADF_EVENT']['outcome'], 'success') + + def test_api_request_failure(self): + middleware = audit.AuditMiddleware(FakeFailingApp(), + audit_map_file= + 'etc/pycadf/api_audit_map.conf', + service_name='pycadf') + self.ENV_HEADERS['REQUEST_METHOD'] = 'GET' + req = webob.Request.blank('/foo/bar', + environ=self.ENV_HEADERS) + with mock.patch('oslo.messaging.Notifier.info') as notify: + try: + middleware(req) + self.fail("Application exception has not been re-raised") + except Exception: + pass + # Check first notification with only 'request' + call_args = notify.call_args_list[0][0] + self.assertEqual(call_args[1], 'http.request') + self.assertEqual(set(call_args[2].keys()), + set(['request'])) + + request = call_args[2]['request'] + self.assertEqual(request['PATH_INFO'], '/foo/bar') + self.assertEqual(request['REQUEST_METHOD'], 'GET') + self.assertIn('CADF_EVENT', request) + self.assertEqual(request['CADF_EVENT']['outcome'], 'pending') + + # Check second notification with request + response + call_args = notify.call_args_list[1][0] + self.assertEqual(call_args[1], 'http.response') + self.assertEqual(set(call_args[2].keys()), + set(['request', 'exception'])) + + request = call_args[2]['request'] + self.assertEqual(request['PATH_INFO'], '/foo/bar') + self.assertEqual(request['REQUEST_METHOD'], 'GET') + self.assertIn('CADF_EVENT', request) + self.assertEqual(request['CADF_EVENT']['outcome'], 'unknown') + + def test_process_request_fail(self): + def func_error(self, req): + raise Exception('error') + self.stubs.Set(cadf_api.OpenStackAuditApi, 'append_audit_event', + func_error) + middleware = audit.AuditMiddleware(FakeApp(), + audit_map_file= + 'etc/pycadf/api_audit_map.conf', + service_name='pycadf') + req = webob.Request.blank('/foo/bar', + environ={'REQUEST_METHOD': 'GET'}) + middleware.process_request(req) + + def test_process_response_fail(self): + def func_error(self, req, res): + raise Exception('error') + self.stubs.Set(cadf_api.OpenStackAuditApi, 'mod_audit_event', + func_error) + middleware = audit.AuditMiddleware(FakeApp(), + audit_map_file= + 'etc/pycadf/api_audit_map.conf', + service_name='pycadf') + req = webob.Request.blank('/foo/bar', + environ={'REQUEST_METHOD': 'GET'}) + middleware.process_response(req, webob.response.Response()) diff --git a/requirements.txt b/requirements.txt index bda142c..f8d73a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ Babel>=1.3 iso8601>=0.1.8 -oslo.config>=1.2.0 netaddr>=0.7.6 +oslo.config>=1.2.0 +oslo.messaging>=1.3.0a4 pytz>=2010h six>=1.4.1 WebOb>=1.2.3 @@ -16,7 +16,6 @@ classifier = Programming Language :: Python Programming Language :: Python :: 2.6 Programming Language :: Python :: 2.7 - Programming Language :: Python :: 3.3 [files] packages = diff --git a/test-requirements.txt b/test-requirements.txt index 6d97af1..0585c98 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,6 +4,8 @@ hacking>=0.8.0,<0.9 coverage>=3.6 discover fixtures>=0.3.14 +mock>=1.0 +mox>=0.5.3 python-subunit>=0.0.18 testrepository>=0.0.17 testscenarios>=0.4 |