Add exception handler to updateBuildDescriptions

Adding a exception handler to updateBuildDescriptions to
handle exceptions from launcher (Gearman).

This addresses the corner case seen in:
https://storyboard.openstack.org/#!/story/2000445

In the above mentioned corner case, Gearman throws a communication
error exception. Since updateBuildDescriptions is called after the
build status has been reported to Gerrit, we want to make sure that
we return back to caller so that the build is properly marked as
reported in reportItem to avoid an infinite loop.

Change-Id: I166d4a2e1d45e97a9ed0b6addb7270e1bf92d6f7

1 files changed, 14 insertions, 4 deletions

diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index f8321d14e..151aae14e 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -1535,13 +1535,23 @@ class BasePipelineManager(object):
 
     def updateBuildDescriptions(self, build_set):
         for build in build_set.getBuilds():
-            desc = self.formatDescription(build)
-            self.sched.launcher.setBuildDescription(build, desc)
+            try:
+                desc = self.formatDescription(build)
+                self.sched.launcher.setBuildDescription(build, desc)
+            except:
+                # Log the failure and let loop continue
+                self.log.error("Failed to update description for build %s" %
+                               (build))
 
         if build_set.previous_build_set:
             for build in build_set.previous_build_set.getBuilds():
-                desc = self.formatDescription(build)
-                self.sched.launcher.setBuildDescription(build, desc)
+                try:
+                    desc = self.formatDescription(build)
+                    self.sched.launcher.setBuildDescription(build, desc)
+                except:
+                    # Log the failure and let loop continue
+                    self.log.error("Failed to update description for "
+                                   "build %s in previous build set" % (build))
 
     def onBuildStarted(self, build):
         self.log.debug("Build %s started" % build)