Cloner: add project-specific branch overrides

Add a facility to override the indicated branch for a specific
project.  In OpenStack, this is used to specify that tempest should
always use its master branch even though it has stable branches
(we no longer use stable branches for tempest).  It may also be
useful in some kinds of compatibility cross-testing.

The test for this is taken from 'test_multi_branch_project_override'
in devstack-gate.

Change-Id: I958af233d1def3b1c7362e1b2ddc77c108766358

3 files changed, 112 insertions, 5 deletions

diff --git a/tests/test_cloner.py b/tests/test_cloner.py
index 752f37c63..166c48666 100644
--- a/tests/test_cloner.py
+++ b/tests/test_cloner.py
@@ -339,3 +339,87 @@ class TestCloner(ZuulTestCase):
         self.worker.hold_jobs_in_build = False
         self.worker.release()
         self.waitUntilSettled()
+
+    def test_project_override(self):
+        self.worker.hold_jobs_in_build = True
+        projects = ['org/project1', 'org/project2', 'org/project3',
+                    'org/project4', 'org/project5', 'org/project6']
+
+        self.create_branch('org/project3', 'stable/havana')
+        self.create_branch('org/project4', 'stable/havana')
+        self.create_branch('org/project6', 'stable/havana')
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+        C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+        D = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana', 'D')
+        A.addApproval('CRVW', 2)
+        B.addApproval('CRVW', 2)
+        C.addApproval('CRVW', 2)
+        D.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
+
+        self.waitUntilSettled()
+
+        self.assertEquals(4, len(self.builds), "Four builds are running")
+
+        upstream = self.getUpstreamRepos(projects)
+        states = [
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('stable/havana')),
+             },
+            ]
+
+        for number, build in enumerate(self.builds):
+            self.log.debug("Build parameters: %s", build.parameters)
+            change_number = int(build.parameters['ZUUL_CHANGE'])
+            cloner = zuul.lib.cloner.Cloner(
+                git_base_url=self.upstream_root,
+                projects=projects,
+                workspace=self.workspace_root,
+                zuul_branch=build.parameters['ZUUL_BRANCH'],
+                zuul_ref=build.parameters['ZUUL_REF'],
+                zuul_url=self.git_root,
+                project_branches={'org/project4': 'master'},
+                )
+            cloner.execute()
+            work = self.getWorkspaceRepos(projects)
+            state = states[number]
+
+            for project in projects:
+                self.assertEquals(state[project],
+                                  str(work[project].commit('HEAD')),
+                                  'Project %s commit for build %s should '
+                                  'be correct' % (project, number))
+            shutil.rmtree(self.workspace_root)
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
diff --git a/zuul/cmd/cloner.py b/zuul/cmd/cloner.py
index 1310c1690..2fcaacd29 100755
--- a/zuul/cmd/cloner.py
+++ b/zuul/cmd/cloner.py
@@ -61,17 +61,24 @@ class Cloner(zuul.cmd.ZuulApp):
 
         project_env = parser.add_argument_group(
             'project tuning'
-        )
+            )
         project_env.add_argument(
             '--branch',
             help=('branch to checkout instead of Zuul selected branch, '
                   'for example to specify an alternate branch to test '
                   'client library compatibility.')
-        )
+            )
+        project_env.add_argument(
+            '--project-branch', nargs=1, action='append',
+            metavar='PROJECT=BRANCH',
+            help=('project-specific branch to checkout which takes precedence '
+                  'over --branch if it is provided; may be specified multiple '
+                  'times.')
+            )
 
         zuul_env = parser.add_argument_group(
             'zuul environnement',
-            'Let you override $ZUUL_* environnement variables.'
+            'Let you override $ZUUL_* environment variables.'
         )
         for zuul_suffix in ZUUL_ENV_SUFFIXES:
             env_name = 'ZUUL_%s' % zuul_suffix.upper()
@@ -120,6 +127,11 @@ class Cloner(zuul.cmd.ZuulApp):
     def main(self):
         self.parse_arguments()
         self.setup_logging(color=self.args.color, verbose=self.args.verbose)
+        project_branches = {}
+        if self.args.project_branch:
+            for x in self.args.project_branch:
+                project, branch = x[0].split('=')
+                project_branches[project] = branch
         cloner = zuul.lib.cloner.Cloner(
             git_base_url=self.args.git_base_url,
             projects=self.args.projects,
@@ -128,7 +140,8 @@ class Cloner(zuul.cmd.ZuulApp):
             zuul_ref=self.args.zuul_ref,
             zuul_url=self.args.zuul_url,
             branch=self.args.branch,
-            clone_map_file=self.args.clone_map_file
+            clone_map_file=self.args.clone_map_file,
+            project_branches=project_branches,
         )
         cloner.execute()
 
diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
index a38281c64..579b9c7e1 100644
--- a/zuul/lib/cloner.py
+++ b/zuul/lib/cloner.py
@@ -28,7 +28,8 @@ class Cloner(object):
     log = logging.getLogger("zuul.Cloner")
 
     def __init__(self, git_base_url, projects, workspace, zuul_branch,
-                 zuul_ref, zuul_url, branch=None, clone_map_file=None):
+                 zuul_ref, zuul_url, branch=None, clone_map_file=None,
+                 project_branches=None):
 
         self.clone_map = []
         self.dests = None
@@ -40,6 +41,7 @@ class Cloner(object):
         self.zuul_branch = zuul_branch
         self.zuul_ref = zuul_ref
         self.zuul_url = zuul_url
+        self.project_branches = project_branches or {}
 
         if clone_map_file:
             self.readCloneMap(clone_map_file)
@@ -98,6 +100,12 @@ class Cloner(object):
          2) Zuul reference for the master branch
          3) The tip of the indicated branch
          4) The tip of the master branch
+
+        The "indicated branch" is one of the following:
+
+         A) The project-specific override branch (from project_branches arg)
+         B) The user specified branch (from the branch arg)
+         C) ZUUL_BRANCH (from the zuul_branch arg)
         """
 
         repo = self.cloneUpstream(project, dest)
@@ -107,6 +115,8 @@ class Cloner(object):
         repo.prune()
 
         indicated_branch = self.branch or self.zuul_branch
+        if project in self.project_branches:
+            indicated_branch = self.project_branches[project]
 
         override_zuul_ref = re.sub(self.zuul_branch, indicated_branch,
                                    self.zuul_ref)