Add processor that sanitizes configurable keys

The logic of the new processor is almost identical to the logic of the
existing SanitizePasswordsProcessor, so this commit also changes the
implementation of SanitizePasswordsProcessor to avoid code-duplication.
It now inherits from the new SanitizeKeysProcessor.

3 files changed, 144 insertions, 18 deletions

diff --git a/raven/base.py b/raven/base.py
index 4fa20f6..9be3a0f 100644
--- a/raven/base.py
+++ b/raven/base.py
@@ -184,6 +184,7 @@ class Client(object):
         self.site = o.get('site')
         self.include_versions = o.get('include_versions', True)
         self.processors = o.get('processors')
+        self.sanitize_keys = o.get('sanitize_keys')
         if self.processors is None:
             self.processors = defaults.PROCESSORS
 
diff --git a/raven/processors.py b/raven/processors.py
index dd3c769..c5b3b41 100644
--- a/raven/processors.py
+++ b/raven/processors.py
@@ -64,32 +64,25 @@ class RemoveStackLocalsProcessor(Processor):
             frame.pop('vars', None)
 
 
-class SanitizePasswordsProcessor(Processor):
+class SanitizeKeysProcessor(Processor):
     """
     Asterisk out things that look like passwords, credit card numbers,
     and API keys in frames, http, and basic extra data.
     """
 
     MASK = '*' * 8
-    FIELDS = frozenset([
-        'password',
-        'secret',
-        'passwd',
-        'authorization',
-        'api_key',
-        'apikey',
-        'sentry_dsn',
-        'access_token',
-    ])
-    VALUES_RE = re.compile(r'^(?:\d[ -]*?){13,16}$')
+
+    def __init__(self, client):
+        super(SanitizeKeysProcessor, self).__init__(client)
+        fields = getattr(client, 'sanitize_keys')
+        if fields is None:
+            raise ValueError('The sanitize_keys setting must be present to use SanitizeKeysProcessor')
+        self.fields = fields
 
     def sanitize(self, key, value):
         if value is None:
             return
 
-        if isinstance(value, string_types) and self.VALUES_RE.match(value):
-            return self.MASK
-
         if not key:  # key can be a NoneType
             return value
 
@@ -101,7 +94,7 @@ class SanitizePasswordsProcessor(Processor):
             key = text_type(key)
 
         key = key.lower()
-        for field in self.FIELDS:
+        for field in self.fields:
             if field in key:
                 # store mask as a fixed length for security
                 return self.MASK
@@ -147,3 +140,27 @@ class SanitizePasswordsProcessor(Processor):
                 sanitized_keyvals.append(keyval)
 
         return delimiter.join('='.join(keyval) for keyval in sanitized_keyvals)
+
+
+class SanitizePasswordsProcessor(SanitizeKeysProcessor):
+    FIELDS = frozenset([
+        'password',
+        'secret',
+        'passwd',
+        'authorization',
+        'api_key',
+        'apikey',
+        'sentry_dsn',
+        'access_token',
+    ])
+    VALUES_RE = re.compile(r'^(?:\d[ -]*?){13,16}$')
+
+    def __init__(self, client):
+        super(SanitizeKeysProcessor, self).__init__(client)  # run the __init__ method of Processor, not SanitizeKeysProcessor
+        self.fields = self.FIELDS
+
+    def sanitize(self, key, value):
+        value = super(SanitizePasswordsProcessor, self).sanitize(key, value)
+        if isinstance(value, string_types) and self.VALUES_RE.match(value):
+            return self.MASK
+        return value
diff --git a/tests/processors/tests.py b/tests/processors/tests.py
index c4c5f52..4c6cb59 100644
--- a/tests/processors/tests.py
+++ b/tests/processors/tests.py
@@ -4,8 +4,8 @@ from mock import Mock
 
 import raven
 from raven.utils.testutils import TestCase
-from raven.processors import SanitizePasswordsProcessor, \
-    RemovePostDataProcessor, RemoveStackLocalsProcessor
+from raven.processors import SanitizeKeysProcessor, \
+    SanitizePasswordsProcessor, RemovePostDataProcessor, RemoveStackLocalsProcessor
 
 
 VARS = {
@@ -21,6 +21,8 @@ VARS = {
     'api_key': 'secret_key',
     'apiKey': 'secret_key',
     'access_token': 'oauth2 access token',
+    'custom_key1': 'you should not see this',
+    'custom_key2': 'you should not see this',
 }
 
 
@@ -33,6 +35,8 @@ def get_stack_trace_data_real(exception_class=TypeError, **kwargs):
         api_key = "I'm hideous!"                # NOQA F841
         apiKey = "4567000012345678"             # NOQA F841
         access_token = "secret stuff!"          # NOQA F841
+        custom_key1 = "you shouldn't see this"  # NOQA F841
+        custom_key2 = "you shouldn't see this"  # NOQA F841
 
         # TypeError: unsupported operand type(s) for /: 'str' and 'str'
         raise exception_class()
@@ -77,6 +81,110 @@ def get_extra_data():
     return data
 
 
+class SanitizeKeysProcessorTest(TestCase):
+
+    def setUp(self):
+        client = Mock(sanitize_keys=['custom_key1', 'custom_key2'])
+        self.proc = SanitizeKeysProcessor(client)
+
+    def _check_vars_sanitized(self, vars, MASK):
+        """
+        Helper to check that keys have been sanitized.
+        """
+        self.assertTrue('custom_key1' in vars)
+        self.assertEquals(vars['custom_key1'], MASK)
+        self.assertTrue('custom_key2' in vars)
+        self.assertEquals(vars['custom_key2'], MASK)
+
+    def test_stacktrace(self, *args, **kwargs):
+        data = get_stack_trace_data_real()
+        result = self.proc.process(data)
+
+        self.assertTrue('exception' in result)
+        exception = result['exception']
+        self.assertTrue('values' in exception)
+        values = exception['values']
+        stack = values[-1]['stacktrace']
+        self.assertTrue('frames' in stack)
+        self.assertEquals(len(stack['frames']), 2)
+        frame = stack['frames'][1]  # frame of will_throw_type_error()
+        self.assertTrue('vars' in frame)
+        self._check_vars_sanitized(frame['vars'], self.proc.MASK)
+
+    def test_http(self):
+        data = get_http_data()
+        result = self.proc.process(data)
+
+        self.assertTrue('request' in result)
+        http = result['request']
+        for n in ('data', 'env', 'headers', 'cookies'):
+            self.assertTrue(n in http)
+            self._check_vars_sanitized(http[n], self.proc.MASK)
+
+    def test_extra(self):
+        data = get_extra_data()
+        result = self.proc.process(data)
+
+        self.assertTrue('extra' in result)
+        extra = result['extra']
+        self._check_vars_sanitized(extra, self.proc.MASK)
+
+    def test_querystring_as_string(self):
+        data = get_http_data()
+        data['request']['query_string'] = 'foo=bar&custom_key1=nope&custom_key2=nope'
+        result = self.proc.process(data)
+
+        self.assertTrue('request' in result)
+        http = result['request']
+        self.assertEquals(
+            http['query_string'],
+            'foo=bar&custom_key1=%(m)s&custom_key2=%(m)s' % {'m': self.proc.MASK})
+
+    def test_querystring_as_string_with_partials(self):
+        data = get_http_data()
+        data['request']['query_string'] = 'foo=bar&custom_key1&baz=bar'
+        result = self.proc.process(data)
+
+        self.assertTrue('request' in result)
+        http = result['request']
+        self.assertEquals(http['query_string'], 'foo=bar&custom_key1&baz=bar' % {'m': self.proc.MASK})
+
+    def test_cookie_as_string(self):
+        data = get_http_data()
+        data['request']['cookies'] = 'foo=bar;custom_key1=nope;custom_key2=nope;'
+        result = self.proc.process(data)
+
+        self.assertTrue('request' in result)
+        http = result['request']
+        self.assertEquals(
+            http['cookies'],
+            'foo=bar;custom_key1=%(m)s;custom_key2=%(m)s;' % {'m': self.proc.MASK})
+
+    def test_cookie_as_string_with_partials(self):
+        data = get_http_data()
+        data['request']['cookies'] = 'foo=bar;custom_key1;baz=bar'
+        result = self.proc.process(data)
+
+        self.assertTrue('request' in result)
+        http = result['request']
+        self.assertEquals(http['cookies'], 'foo=bar;custom_key1;baz=bar' % dict(m=self.proc.MASK))
+
+    def test_cookie_header(self):
+        data = get_http_data()
+        data['request']['headers']['Cookie'] = 'foo=bar;custom_key1=nope;custom_key2=nope;'
+        result = self.proc.process(data)
+
+        self.assertTrue('request' in result)
+        http = result['request']
+        self.assertEquals(
+            http['headers']['Cookie'],
+            'foo=bar;custom_key1=%(m)s;custom_key2=%(m)s;' % {'m': self.proc.MASK})
+
+    def test_sanitize_non_ascii(self):
+        result = self.proc.sanitize('__repr__: жили-были', '42')
+        self.assertEquals(result, '42')
+
+
 class SanitizePasswordsProcessorTest(TestCase):
 
     def _check_vars_sanitized(self, vars, proc):