Add basic deduplication for log messages.

3 files changed, 48 insertions, 6 deletions

diff --git a/CHANGES b/CHANGES
index 6a0fdeb..d4ad800 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,9 @@
+Version 5.18.0
+--------------
+
+* Breadcrumbs are now attempted to be deduplicated to catch some common
+  cases where log messages just spam up the breadcrumbs.
+
 Version 5.17.0
 --------------
 
diff --git a/raven/breadcrumbs.py b/raven/breadcrumbs.py
index 803c9c5..6d298e9 100644
--- a/raven/breadcrumbs.py
+++ b/raven/breadcrumbs.py
@@ -14,20 +14,29 @@ special_logger_handlers = {}
 logger = logging.getLogger('raven')
 
 
+def event_payload_considered_equal(a, b):
+    return (
+        a['type'] == b['type'] and
+        a['level'] == b['level'] and
+        a['message'] == b['message'] and
+        a['category'] == b['category'] and
+        a['data'] == b['data']
+    )
+
+
 class BreadcrumbBuffer(object):
 
     def __init__(self, limit=100):
         self.buffer = []
         self.limit = limit
 
-    def record(self, type, timestamp=None, duration=None, level=None,
-               message=None, category=None, data=None, processor=None):
+    def record(self, type, timestamp=None, level=None, message=None,
+               category=None, data=None, processor=None):
         if timestamp is None:
             timestamp = time.time()
         self.buffer.append(({
             'type': type,
             'timestamp': timestamp,
-            'duration': duration,
             'level': level,
             'message': message,
             'category': category,
@@ -46,8 +55,10 @@ class BreadcrumbBuffer(object):
                     processor(payload)
                 except Exception:
                     logger.exception('Failed to process breadcrumbs. Ignored')
+                    payload = None
                 self.buffer[idx] = (payload, None)
-            if payload is not None:
+            if payload is not None and \
+               (not rv or not event_payload_considered_equal(rv[-1], payload)):
                 rv.append(payload)
         return rv
 
@@ -63,7 +74,7 @@ def make_buffer(enabled=True):
     return BlackholeBreadcrumbBuffer()
 
 
-def record_breadcrumb(type, timestamp=None, duration=None, level=None,
+def record_breadcrumb(type, timestamp=None, level=None,
                       message=None, category=None, data=None,
                       processor=None):
     """Records a breadcrumb for all active clients.  This is what integration
@@ -73,7 +84,7 @@ def record_breadcrumb(type, timestamp=None, duration=None, level=None,
     if timestamp is None:
         timestamp = time.time()
     for ctx in raven.context.get_active_contexts():
-        ctx.breadcrumbs.record(type, timestamp, duration, level, message,
+        ctx.breadcrumbs.record(type, timestamp, level, message,
                                category, data, processor)
 
 
diff --git a/tests/breadcrumbs/tests.py b/tests/breadcrumbs/tests.py
index 00fd463..79b814b 100644
--- a/tests/breadcrumbs/tests.py
+++ b/tests/breadcrumbs/tests.py
@@ -68,3 +68,28 @@ class BreadcrumbTestCase(TestCase):
         assert crumbs[0]['type'] == 'default'
         assert crumbs[0]['category'] == 'whatever.foo'
         assert crumbs[0]['message'] == 'This is a message with %s. %s!'
+
+    def test_dedup_logging(self):
+        client = Client('http://foo:bar@example.com/0')
+        with client.context:
+            log = logging.getLogger('whatever.foo')
+            log.info('This is a message with %s!', 42)
+            log.info('This is a message with %s!', 42)
+            log.info('This is a message with %s!', 42)
+            log.info('This is a message with %s!', 23)
+            log.info('This is a message with %s!', 23)
+            log.info('This is a message with %s!', 23)
+            log.info('This is a message with %s!', 42)
+            crumbs = client.context.breadcrumbs.get_buffer()
+
+        print crumbs
+        assert len(crumbs) == 3
+        assert crumbs[0]['type'] == 'default'
+        assert crumbs[0]['category'] == 'whatever.foo'
+        assert crumbs[0]['message'] == 'This is a message with 42!'
+        assert crumbs[1]['type'] == 'default'
+        assert crumbs[1]['category'] == 'whatever.foo'
+        assert crumbs[1]['message'] == 'This is a message with 23!'
+        assert crumbs[2]['type'] == 'default'
+        assert crumbs[2]['category'] == 'whatever.foo'
+        assert crumbs[2]['message'] == 'This is a message with 42!'