Do not try to save event resource if too big for db column

Change-Id: Idedcfb5bf6678cd24990d1503c223ea9a9ac41cf
Closes-Bug: #1552431
(cherry picked from commit b48452fc9acd4e4c9f8fd83ca1fc9782635cec36)

2 files changed, 95 insertions, 32 deletions

diff --git a/heat/engine/event.py b/heat/engine/event.py
index 7cf03175f..221c6df53 100644
--- a/heat/engine/event.py
+++ b/heat/engine/event.py
@@ -11,15 +11,21 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import pickle
 import six
 
 import oslo_db.exception
+from oslo_log import log as logging
 
 from heat.common import exception
 from heat.common.i18n import _
 from heat.common import identifier
 from heat.objects import event as event_object
 
+LOG = logging.getLogger(__name__)
+
+MAX_EVENT_RESOURCE_PROPERTIES_SIZE = (1 << 16) - 1
+
 
 class Event(object):
     """Class representing a Resource state change."""
@@ -87,22 +93,38 @@ class Event(object):
         if self.timestamp is not None:
             ev['created_at'] = self.timestamp
 
-        try:
-            new_ev = event_object.Event.create(self.context, ev)
-        except oslo_db.exception.DBError:
-            # Attempt do drop the largest key and re-store as we expect
-            # This to mostly happen with one large config blob property
+        # Workaround: we don't want to attempt to store the
+        # event.resource_properties column if the data is too large
+        # (greater than permitted by BLOB). Otherwise, we end up with
+        # an unsightly log message.
+        rp_size = len(pickle.dumps(ev['resource_properties']))
+        if rp_size > MAX_EVENT_RESOURCE_PROPERTIES_SIZE:
+            LOG.debug('event\'s resource_properties too large to store at '
+                      '%d bytes', rp_size)
+            # Try truncating the largest value and see if that gets us under
+            # the db column's size constraint.
             max_key, max_val = max(ev['resource_properties'].items(),
                                    key=lambda i: len(repr(i[1])))
-            err = 'Resource properties are too large to store'
+            err = 'Resource properties are too large to store fully'
             ev['resource_properties'].update({'Error': err})
             ev['resource_properties'][max_key] = '<Deleted, too large>'
-            try:
-                new_ev = event_object.Event.create(self.context, ev)
-            except oslo_db.exception.DBError:
-                # Give up and drop all properties..
+            rp_size = len(pickle.dumps(ev['resource_properties']))
+            if rp_size > MAX_EVENT_RESOURCE_PROPERTIES_SIZE:
+                LOG.debug('event\'s resource_properties STILL too large '
+                          'after truncating largest key at %d bytes', rp_size)
+                err = 'Resource properties are too large to attempt to store'
                 ev['resource_properties'] = {'Error': err}
-                new_ev = event_object.Event.create(self.context, ev)
+
+        # We should have worked around the issue, but let's be extra
+        # careful.
+        try:
+            new_ev = event_object.Event.create(self.context, ev)
+        except oslo_db.exception.DBError:
+            # Give up and drop all properties..
+            err = 'Resource properties are too large to store'
+            ev['resource_properties'] = {'Error': err}
+            new_ev = event_object.Event.create(self.context, ev)
+
         self.id = new_ev.id
         return self.id
 
diff --git a/heat/tests/test_event.py b/heat/tests/test_event.py
index 77787c9dc..4e4e5d2a6 100644
--- a/heat/tests/test_event.py
+++ b/heat/tests/test_event.py
@@ -52,6 +52,30 @@ tmpl_multiple = {
     }
 }
 
+tmpl_multiple_too_large = {
+    'HeatTemplateFormatVersion': '2012-12-12',
+    'Resources': {
+        'EventTestResource': {
+            'Type': 'ResourceWithMultipleRequiredProps',
+            'Properties': {'Foo1': 'zoo',
+                           'Foo2': 'A' * (1 << 16),
+                           'Foo3': '99999'}
+        }
+    }
+}
+
+tmpl_multiple_srsly_too_large = {
+    'HeatTemplateFormatVersion': '2012-12-12',
+    'Resources': {
+        'EventTestResource': {
+            'Type': 'ResourceWithMultipleRequiredProps',
+            'Properties': {'Foo1': 'Z' * (1 << 16),
+                           'Foo2': 'A' * (1 << 16),
+                           'Foo3': '99999'}
+        }
+    }
+}
+
 
 class EventCommon(common.HeatTestCase):
 
@@ -203,13 +227,13 @@ class EventTest(EventCommon):
         self.assertIn('Error', e.resource_properties)
 
 
-class EventTestProps(EventCommon):
+class EventTestSingleLargeProp(EventCommon):
 
     def setUp(self):
-        super(EventTestProps, self).setUp()
-        self._setup_stack(tmpl_multiple)
+        super(EventTestSingleLargeProp, self).setUp()
+        self._setup_stack(tmpl_multiple_too_large)
 
-    def test_store_fail_all_props(self):
+    def test_too_large_single_prop(self):
         self.resource.resource_id_set('resource_physical_id')
 
         e = event.Event(self.ctx, self.stack, 'TEST', 'IN_PROGRESS', 'Testing',
@@ -219,22 +243,42 @@ class EventTestProps(EventCommon):
         self.assertIsNotNone(e.id)
         ev = event_object.Event.get_by_id(self.ctx, e.id)
 
-        errors = [oslo_db.exception.DBError, oslo_db.exception.DBError]
+        self.assertEqual(
+            {'Foo1': 'zoo',
+             'Foo2': '<Deleted, too large>',
+             'Foo3': '99999',
+             'Error': 'Resource properties are too large to store fully'},
+            ev['resource_properties'])
 
-        def side_effect(*args):
-            try:
-                raise errors.pop()
-            except IndexError:
-                self.assertEqual(
-                    {'Error': 'Resource properties are too large to store'},
-                    args[1]['resource_properties'])
-                return ev
 
-        with mock.patch("heat.objects.event.Event") as mock_event:
-            mock_event.create.side_effect = side_effect
-            e.store()
+class EventTestMultipleLargeProp(EventCommon):
 
-    def test_store_fail_one_prop(self):
+    def setUp(self):
+        super(EventTestMultipleLargeProp, self).setUp()
+        self._setup_stack(tmpl_multiple_srsly_too_large)
+
+    def test_too_large_multiple_prop(self):
+        self.resource.resource_id_set('resource_physical_id')
+
+        e = event.Event(self.ctx, self.stack, 'TEST', 'IN_PROGRESS', 'Testing',
+                        'alabama', self.resource.properties,
+                        self.resource.name, self.resource.type())
+        e.store()
+        self.assertIsNotNone(e.id)
+        ev = event_object.Event.get_by_id(self.ctx, e.id)
+
+        self.assertEqual(
+            {'Error': 'Resource properties are too large to attempt to store'},
+            ev['resource_properties'])
+
+
+class EventTestStoreProps(EventCommon):
+
+    def setUp(self):
+        super(EventTestStoreProps, self).setUp()
+        self._setup_stack(tmpl_multiple)
+
+    def test_store_fail_all_props(self):
         self.resource.resource_id_set('resource_physical_id')
 
         e = event.Event(self.ctx, self.stack, 'TEST', 'IN_PROGRESS', 'Testing',
@@ -251,10 +295,7 @@ class EventTestProps(EventCommon):
                 raise errors.pop()
             except IndexError:
                 self.assertEqual(
-                    {'Foo1': 'zoo',
-                     'Foo2': '<Deleted, too large>',
-                     'Foo3': '99999',
-                     'Error': 'Resource properties are too large to store'},
+                    {'Error': 'Resource properties are too large to store'},
                     args[1]['resource_properties'])
                 return ev