Support project templates

Change-Id: I37378893e1761fe33c19d936076b9ec3d4c81627

9 files changed, 305 insertions, 87 deletions

diff --git a/tests/fixtures/config/multi-tenant/tenant-one.yaml b/tests/fixtures/config/multi-tenant/tenant-one.yaml
index c9096ef1c..874e932fb 100644
--- a/tests/fixtures/config/multi-tenant/tenant-one.yaml
+++ b/tests/fixtures/config/multi-tenant/tenant-one.yaml
@@ -28,6 +28,8 @@ jobs:
 projects:
   - name: org/project1
     check:
-      - project1-test1
+      jobs:
+        - project1-test1
     tenant-one-gate:
-      - project1-test1
+      jobs:
+        - project1-test1
diff --git a/tests/fixtures/config/multi-tenant/tenant-two.yaml b/tests/fixtures/config/multi-tenant/tenant-two.yaml
index 6cb2d9a52..254d9cdba 100644
--- a/tests/fixtures/config/multi-tenant/tenant-two.yaml
+++ b/tests/fixtures/config/multi-tenant/tenant-two.yaml
@@ -28,6 +28,8 @@ jobs:
 projects:
   - name: org/project2
     check:
-      - project2-test1
+      jobs:
+        - project2-test1
     tenant-two-gate:
-      - project2-test1
+      jobs:
+        - project2-test1
diff --git a/tests/fixtures/config/project-template/common.yaml b/tests/fixtures/config/project-template/common.yaml
new file mode 100644
index 000000000..9e76bde07
--- /dev/null
+++ b/tests/fixtures/config/project-template/common.yaml
@@ -0,0 +1,56 @@
+pipelines:
+  - name: check
+    manager: independent
+    source:
+      gerrit
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+  - name: gate
+    manager: dependent
+    success-message: Build succeeded (gate).
+    source:
+      gerrit
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - approved: 1
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    start:
+      gerrit:
+        verified: 0
+    precedence: high
+
+jobs:
+  - name:
+      project-test1
+  - name:
+      project-test2
+
+project-templates:
+  - name: test-template
+    gate:
+      jobs:
+        - project-test2
+
+projects:
+  - name: org/project
+    templates:
+      - test-template
+    gate:
+      jobs:
+        - project-test1
diff --git a/tests/fixtures/config/project-template/main.yaml b/tests/fixtures/config/project-template/main.yaml
new file mode 100644
index 000000000..c89fdfaff
--- /dev/null
+++ b/tests/fixtures/config/project-template/main.yaml
@@ -0,0 +1,4 @@
+tenants:
+  - name: tenant-one
+    include:
+      - common.yaml
diff --git a/tests/fixtures/config/project-template/tenant-one.yaml b/tests/fixtures/config/project-template/tenant-one.yaml
new file mode 100644
index 000000000..c9096ef1c
--- /dev/null
+++ b/tests/fixtures/config/project-template/tenant-one.yaml
@@ -0,0 +1,33 @@
+pipelines:
+  - name: tenant-one-gate
+    manager: dependent
+    success-message: Build succeeded (tenant-one-gate).
+    source:
+      gerrit
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - approved: 1
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    start:
+      gerrit:
+        verified: 0
+    precedence: high
+
+jobs:
+  - name:
+      project1-test1
+
+projects:
+  - name: org/project1
+    check:
+      - project1-test1
+    tenant-one-gate:
+      - project1-test1
diff --git a/tests/fixtures/config/project-template/zuul.conf b/tests/fixtures/config/project-template/zuul.conf
new file mode 100644
index 000000000..67f3d2c26
--- /dev/null
+++ b/tests/fixtures/config/project-template/zuul.conf
@@ -0,0 +1,36 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+tenant_config=config/project-template/main.yaml
+url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
+job_name_in_report=true
+
+[merger]
+git_dir=/tmp/zuul-test/git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[swift]
+authurl=https://identity.api.example.org/v2.0/
+user=username
+key=password
+tenant_name=" "
+
+default_container=logs
+region_name=EXP
+logserver_prefix=http://logs.example.org/server.app/
+
+[connection gerrit]
+driver=gerrit
+server=review.example.com
+user=jenkins
+sshkey=none
+
+[connection smtp]
+driver=smtp
+server=localhost
+port=25
+default_from=zuul@example.com
+default_to=you@example.com
diff --git a/tests/test_v3.py b/tests/test_v3.py
index 73efcc99d..b746eae6a 100644
--- a/tests/test_v3.py
+++ b/tests/test_v3.py
@@ -78,7 +78,8 @@ class TestInRepoConfig(ZuulTestCase):
             projects:
               - name: org/project
                 tenant-one-gate:
-                  - project-test1
+                  jobs:
+                    - project-test1
             """)
 
         self.addCommitToRepo('org/project', 'add zuul conf',
@@ -96,3 +97,22 @@ class TestInRepoConfig(ZuulTestCase):
                          "A should report start and success")
         self.assertIn('tenant-one-gate', A.messages[1],
                       "A should transit tenant-one gate")
+
+
+class TestProjectTemplate(ZuulTestCase):
+    config_file = 'config/project-template/zuul.conf'
+
+    def test(self):
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        A.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+        self.waitUntilSettled()
+        self.assertEqual(self.getJobFromHistory('project-test1').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory('project-test2').result,
+                         'SUCCESS')
+        self.assertEqual(A.data['status'], 'MERGED')
+        self.assertEqual(A.reported, 2,
+                         "A should report start and success")
+        self.assertIn('gate', A.messages[1],
+                      "A should transit gate")
diff --git a/zuul/configloader.py b/zuul/configloader.py
index d22106d95..d09ba852d 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -150,6 +150,107 @@ class JobParser(object):
         return job
 
 
+class ProjectTemplateParser(object):
+    log = logging.getLogger("zuul.ProjectTemplateParser")
+
+    @staticmethod
+    def getSchema(layout):
+        project_template = {vs.Required('name'): str}
+        for p in layout.pipelines.values():
+            project_template[p.name] = {'queue': str,
+                                        'jobs': [vs.Any(str, dict)]}
+        return vs.Schema(project_template)
+
+    @staticmethod
+    def fromYaml(layout, conf):
+        ProjectTemplateParser.getSchema(layout)(conf)
+        project_template = model.ProjectConfig(conf['name'])
+        for pipeline in layout.pipelines.values():
+            conf_pipeline = conf.get(pipeline.name)
+            if not conf_pipeline:
+                continue
+            project_pipeline = model.ProjectPipelineConfig()
+            project_template.pipelines[pipeline.name] = project_pipeline
+            project_pipeline.queue_name = conf.get('queue')
+            project_pipeline.job_tree = ProjectTemplateParser._parseJobTree(
+                layout, conf_pipeline.get('jobs'))
+        return project_template
+
+    @staticmethod
+    def _parseJobTree(layout, conf, tree=None):
+        if not tree:
+            tree = model.JobTree(None)
+        for conf_job in conf:
+            if isinstance(conf_job, basestring):
+                tree.addJob(layout.getJob(conf_job))
+            elif isinstance(conf_job, dict):
+                # A dictionary in a job tree may override params, or
+                # be the root of a sub job tree, or both.
+                jobname, attrs = dict.items()[0]
+                jobs = attrs.pop('jobs')
+                if attrs:
+                    # We are overriding params, so make a new job def
+                    attrs['name'] = jobname
+                    subtree = tree.addJob(JobParser.fromYaml(layout, attrs))
+                else:
+                    # Not overriding, so get existing job
+                    subtree = tree.addJob(layout.getJob(jobname))
+
+                if jobs:
+                    # This is the root of a sub tree
+                    ProjectTemplateParser._parseJobTree(layout, jobs, subtree)
+            else:
+                raise Exception("Job must be a string or dictionary")
+        return tree
+
+
+class ProjectParser(object):
+    log = logging.getLogger("zuul.ProjectParser")
+
+    @staticmethod
+    def getSchema(layout):
+        project = {vs.Required('name'): str,
+                   'templates': [str]}
+        for p in layout.pipelines.values():
+            project[p.name] = {'queue': str,
+                               'jobs': [vs.Any(str, dict)]}
+        return vs.Schema(project)
+
+    @staticmethod
+    def fromYaml(layout, conf):
+        ProjectParser.getSchema(layout)(conf)
+        conf_templates = conf.pop('templates', [])
+        # The way we construct a project definition is by parsing the
+        # definition as a template, then applying all of the
+        # templates, including the newly parsed one, in order.
+        project_template = ProjectTemplateParser.fromYaml(layout, conf)
+        configs = [layout.project_templates[name] for name in conf_templates]
+        configs.append(project_template)
+        project = model.ProjectConfig(conf['name'])
+        for pipeline in layout.pipelines.values():
+            project_pipeline = model.ProjectPipelineConfig()
+            project_pipeline.job_tree = model.JobTree(None)
+            queue_name = None
+            # For every template, iterate over the job tree and replace or
+            # create the jobs in the final definition as needed.
+            pipeline_defined = False
+            for template in configs:
+                ProjectParser.log.debug("Applying template %s to pipeline %s" %
+                                        (template.name, pipeline.name))
+                if pipeline.name in template.pipelines:
+                    pipeline_defined = True
+                    template_pipeline = template.pipelines[pipeline.name]
+                    project_pipeline.job_tree.inheritFrom(
+                        template_pipeline.job_tree)
+                    if template_pipeline.queue_name:
+                        queue_name = template_pipeline.queue_name
+            if queue_name:
+                project_pipeline.queue_name = queue_name
+            if pipeline_defined:
+                project.pipelines[pipeline.name] = project_pipeline
+        return project
+
+
 class AbideValidator(object):
     tenant_source = vs.Schema({'repos': [str]})
 
@@ -229,7 +330,6 @@ class ConfigLoader(object):
 
     def _parseLayout(self, base, data, scheduler, connections):
         layout = model.Layout()
-        project_templates = {}
 
         # TODOv3(jeblair): add validation
         # validator = layoutvalidator.LayoutValidator()
@@ -329,63 +429,16 @@ class ConfigLoader(object):
                 manager.event_filters += trigger.getEventFilters(
                     conf_pipeline['trigger'][trigger_name])
 
-        for project_template in data.get('project-templates', []):
-            # Make sure the template only contains valid pipelines
-            tpl = dict(
-                (pipe_name, project_template.get(pipe_name))
-                for pipe_name in layout.pipelines.keys()
-                if pipe_name in project_template
-            )
-            project_templates[project_template.get('name')] = tpl
-
         for config_job in data.get('jobs', []):
             layout.addJob(JobParser.fromYaml(layout, config_job))
 
-        def add_jobs(job_tree, config_jobs):
-            for job in config_jobs:
-                if isinstance(job, list):
-                    for x in job:
-                        add_jobs(job_tree, x)
-                if isinstance(job, dict):
-                    for parent, children in job.items():
-                        parent_tree = job_tree.addJob(layout.getJob(parent))
-                        add_jobs(parent_tree, children)
-                if isinstance(job, str):
-                    job_tree.addJob(layout.getJob(job))
+        for config_template in data.get('project-templates', []):
+            layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
+                layout, config_template))
 
         for config_project in data.get('projects', []):
-            shortname = config_project['name'].split('/')[-1]
-
-            # This is reversed due to the prepend operation below, so
-            # the ultimate order is templates (in order) followed by
-            # statically defined jobs.
-            for requested_template in reversed(
-                config_project.get('template', [])):
-                # Fetch the template from 'project-templates'
-                tpl = project_templates.get(
-                    requested_template.get('name'))
-                # Expand it with the project context
-                requested_template['name'] = shortname
-                expanded = deep_format(tpl, requested_template)
-                # Finally merge the expansion with whatever has been
-                # already defined for this project.  Prepend our new
-                # jobs to existing ones (which may have been
-                # statically defined or defined by other templates).
-                for pipeline in layout.pipelines.values():
-                    if pipeline.name in expanded:
-                        config_project.update(
-                            {pipeline.name: expanded[pipeline.name] +
-                             config_project.get(pipeline.name, [])})
-
-            mode = config_project.get('merge-mode', 'merge-resolve')
-            for pipeline in layout.pipelines.values():
-                if pipeline.name in config_project:
-                    project = pipeline.source.getProject(
-                        config_project['name'])
-                    project.merge_mode = model.MERGER_MAP[mode]
-                    job_tree = pipeline.addProject(project)
-                    config_jobs = config_project[pipeline.name]
-                    add_jobs(job_tree, config_jobs)
+            layout.addProjectConfig(ProjectParser.fromYaml(
+                layout, config_project))
 
         for pipeline in layout.pipelines.values():
             pipeline.manager._postConfig(layout)
@@ -419,31 +472,3 @@ class ConfigLoader(object):
         # TODOv3(jeblair): this should implement some rules to protect
         # aspects of the config that should not be changed in-repo
         return yaml.load(data)
-
-    def _parseSkipIf(self, config_job):
-        cm = change_matcher
-        skip_matchers = []
-
-        for config_skip in config_job.get('skip-if', []):
-            nested_matchers = []
-
-            project_regex = config_skip.get('project')
-            if project_regex:
-                nested_matchers.append(cm.ProjectMatcher(project_regex))
-
-            branch_regex = config_skip.get('branch')
-            if branch_regex:
-                nested_matchers.append(cm.BranchMatcher(branch_regex))
-
-            file_regexes = to_list(config_skip.get('all-files-match-any'))
-            if file_regexes:
-                file_matchers = [cm.FileMatcher(x) for x in file_regexes]
-                all_files_matcher = cm.MatchAllFiles(file_matchers)
-                nested_matchers.append(all_files_matcher)
-
-            # All patterns need to match a given skip-if predicate
-            skip_matchers.append(cm.MatchAll(nested_matchers))
-
-        if skip_matchers:
-            # Any skip-if predicate can be matched to trigger a skip
-            return cm.MatchAny(skip_matchers)
diff --git a/zuul/model.py b/zuul/model.py
index aa21f859c..a0bacfdc5 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -550,6 +550,17 @@ class JobTree(object):
                 return ret
         return None
 
+    def inheritFrom(self, other):
+        if other.job:
+            self.job = Job(other.job.name)
+            self.job.inheritFrom(other.job)
+        for other_tree in other.job_trees:
+            this_tree = self.getJobTreeForJob(other_tree.job)
+            if not this_tree:
+                this_tree = JobTree(None)
+                self.job_trees.append(this_tree)
+            this_tree.inheritFrom(other_tree)
+
 
 class Build(object):
     def __init__(self, job, uuid):
@@ -1356,9 +1367,26 @@ class ChangeishFilter(BaseFilter):
         return True
 
 
+class ProjectPipelineConfig(object):
+    # Represents a project cofiguration in the context of a pipeline
+    def __init__(self):
+        self.job_tree = None
+        self.queue_name = None
+        # TODOv3(jeblair): add merge mode
+
+
+class ProjectConfig(object):
+    # Represents a project cofiguration
+    def __init__(self, name):
+        self.name = name
+        self.pipelines = {}
+
+
 class Layout(object):
     def __init__(self):
         self.projects = {}
+        self.project_configs = {}
+        self.project_templates = {}
         self.pipelines = OrderedDict()
         # This is a dictionary of name -> [jobs].  The first element
         # of the list is the first job added with that name.  It is
@@ -1371,7 +1399,7 @@ class Layout(object):
     def getJob(self, name):
         if name in self.jobs:
             return self.jobs[name][0]
-        return None
+        raise Exception("Job %s not defined" % (name,))
 
     def getJobs(self, name):
         return self.jobs.get(name, [])
@@ -1385,6 +1413,18 @@ class Layout(object):
     def addPipeline(self, pipeline):
         self.pipelines[pipeline.name] = pipeline
 
+    def addProjectTemplate(self, project_template):
+        self.project_templates[project_template.name] = project_template
+
+    def addProjectConfig(self, project_config):
+        self.project_configs[project_config.name] = project_config
+        # TODOv3(jeblair): tidy up the relationship between pipelines
+        # and projects and projectconfigs
+        for pipeline_name, pipeline_config in project_config.pipelines.items():
+            pipeline = self.pipelines[pipeline_name]
+            project = pipeline.source.getProject(project_config.name)
+            pipeline.job_trees[project] = pipeline_config.job_tree
+
 
 class Tenant(object):
     def __init__(self, name):