Run jobs when their own config changes

This causes file matchers to automatically match when the
configuration of the job itself changes.  This can be used instead
of matching "^.zuul.yaml$" which may cause too many jobs to run
in larger repos.

Change-Id: Ieddaead91b597282c5674ba99b0c0f387843c722

1 files changed, 62 insertions, 15 deletions

diff --git a/zuul/model.py b/zuul/model.py
index c01135aea..d6ec52c9f 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1141,6 +1141,7 @@ class Job(ConfigObject):
             branch_matcher=None,
             file_matcher=None,
             irrelevant_file_matcher=None,  # skip-if
+            match_on_config_updates=True,
             tags=frozenset(),
             provides=frozenset(),
             requires=frozenset(),
@@ -1253,6 +1254,7 @@ class Job(ConfigObject):
         d['cleanup_run'] = list(map(lambda x: x.toSchemaDict(),
                                     self.cleanup_run))
         d['post_review'] = self.post_review
+        d['match_on_config_updates'] = self.match_on_config_updates
         if self.isBase():
             d['parent'] = None
         elif self.parent:
@@ -2111,6 +2113,7 @@ class QueueItem(object):
         self.layout = None
         self.project_pipeline_config = None
         self.job_graph = None
+        self._old_job_graph = None  # Cached job graph of previous layout
         self._cached_sql_results = {}
         self.event = event  # The trigger event that lead to this queue item
 
@@ -2131,6 +2134,7 @@ class QueueItem(object):
         self.layout = None
         self.project_pipeline_config = None
         self.job_graph = None
+        self._old_job_graph = None
 
     def addBuild(self, build):
         self.current_build_set.addBuild(build)
@@ -2176,6 +2180,7 @@ class QueueItem(object):
         except Exception:
             self.project_pipeline_config = None
             self.job_graph = None
+            self._old_job_graph = None
             raise
 
     def hasJobGraph(self):
@@ -2864,6 +2869,31 @@ class QueueItem(object):
                     newrev=newrev,
                     )
 
+    def updatesJobConfig(self, job):
+        log = self.annotateLogger(self.log)
+        layout_ahead = self.pipeline.tenant.layout
+        if self.item_ahead and self.item_ahead.layout:
+            layout_ahead = self.item_ahead.layout
+        if layout_ahead and self.layout and self.layout is not layout_ahead:
+            # This change updates the layout.  Calculate the job as it
+            # would be if the layout had not changed.
+            if self._old_job_graph is None:
+                ppc = layout_ahead.getProjectPipelineConfig(self)
+                log.debug("Creating job graph for config change detecction")
+                self._old_job_graph = layout_ahead.createJobGraph(
+                    self, ppc, skip_file_matcher=True)
+                log.debug("Done creating job graph for "
+                          "config change detecction")
+            old_job = self._old_job_graph.jobs.get(job.name)
+            if old_job is None:
+                log.debug("Found a newly created job")
+                return True  # A newly created job
+            if (job.toDict(self.pipeline.tenant) !=
+                old_job.toDict(self.pipeline.tenant)):
+                log.debug("Found an updated job")
+                return True  # This job's configuration has changed
+        return False
+
 
 class Ref(object):
     """An existing state of a Project."""
@@ -3942,6 +3972,17 @@ class Layout(object):
                     frozen_job.applyVariant(variant, item.layout)
                     frozen_job.name = variant.name
             frozen_job.name = jobname
+
+            # Now merge variables set from this parent ppc
+            # (i.e. project+templates) directly into the job vars
+            frozen_job.updateProjectVariables(ppc.variables)
+
+            # If the job does not specify an ansible version default to the
+            # tenant default.
+            if not frozen_job.ansible_version:
+                frozen_job.ansible_version = \
+                    item.layout.tenant.default_ansible_version
+
             log.debug("Froze job %s for %s", jobname, change)
             # Whether the change matches any of the project pipeline
             # variants
@@ -3965,13 +4006,29 @@ class Layout(object):
                 item.debug("No matching pipeline variants for {jobname}".
                            format(jobname=jobname), indent=2)
                 continue
+            updates_job_config = False
             if not skip_file_matcher and \
                not frozen_job.changeMatchesFiles(change):
-                log.debug("Job %s did not match files in %s",
-                          repr(frozen_job), change)
-                item.debug("Job {jobname} did not match files".
-                           format(jobname=jobname), indent=2)
-                continue
+                matched_files = False
+                if frozen_job.match_on_config_updates:
+                    updates_job_config = item.updatesJobConfig(frozen_job)
+            else:
+                matched_files = True
+            if not matched_files:
+                if updates_job_config:
+                    # Log the reason we're ignoring the file matcher
+                    log.debug("The configuration of job %s is "
+                              "changed by %s; ignoring file matcher",
+                              repr(frozen_job), change)
+                    item.debug("The configuration of job {jobname} is "
+                               "changed; ignoring file matcher".
+                               format(jobname=jobname), indent=2)
+                else:
+                    log.debug("Job %s did not match files in %s",
+                              repr(frozen_job), change)
+                    item.debug("Job {jobname} did not match files".
+                               format(jobname=jobname), indent=2)
+                    continue
             if frozen_job.abstract:
                 raise Exception("Job %s is abstract and may not be "
                                 "directly run" %
@@ -3989,16 +4046,6 @@ class Layout(object):
                 raise Exception("Job %s does not specify a run playbook" % (
                     frozen_job.name,))
 
-            # Now merge variables set from this parent ppc
-            # (i.e. project+templates) directly into the job vars
-            frozen_job.updateProjectVariables(ppc.variables)
-
-            # If the job does not specify an ansible version default to the
-            # tenant default.
-            if not frozen_job.ansible_version:
-                frozen_job.ansible_version = \
-                    item.layout.tenant.default_ansible_version
-
             job_graph.addJob(frozen_job)
 
     def createJobGraph(self, item, ppc, skip_file_matcher=False):