Cloner: default to ZUUL_BRANCH

The cloner takes an argument for the "indicated branch", however,
that argument has a default value of None, rendering it optional.

In the original setup method from devstack gate, that argument
was not optional, instead, it was typically set to ZUUL_BRANCH
unless it needed to be specified otherwise, for example for grenade.

Effectively, if omitted the (optional) branch argument to the cloner
it would use the 'master' branch in preference to the ZUUL_BRANCH
for projects that do not have a zuul ref.  This is not the desired
behavior in the simple case (or quite likely any case), and therefore
should not be the default behavior for the cloner.

Resolve this by having the cloner treat the indicated branch as
ZUUL_BRANCH unless it is specified in the branch argument.  In doing
this, the logic in the cloner is slightly simplified and now more
closely matches that in the setup_project function in devstack-gate.

Rename the cloner test to 'test_one_branch' to match the pattern from
devstack-gate tests.  This test encompases the 'test_one_on_master'
and 'test_two_on_master' tests from d-g.  They are combined because
in these tests we are able to iterate over the Zuul builds and check
the state for each build in turn, which is a more complete form of
testing than what was employed in d-g.

Add the 'test_multi_branch' test to the cloner.  This encompases
'test_multi_branch_on_master' (with an additional fourth project
with no changes), as well as 'test_multi_branch_on_stable'.

Change-Id: Ib26bced46073bd61298c52c836a20085511201f3

4 files changed, 156 insertions, 47 deletions

diff --git a/tests/base.py b/tests/base.py
index 753bc5e60..4ba88fb23 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -56,6 +56,7 @@ import zuul.trigger.zuultrigger
 
 FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
                            'fixtures')
+USE_TEMPDIR = True
 
 logging.basicConfig(level=logging.DEBUG,
                     format='%(asctime)s %(name)-32s '
@@ -165,7 +166,7 @@ class FakeChange(object):
         if files:
             fn = files[0]
         else:
-            fn = '%s-%s' % (self.branch, self.number)
+            fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
         msg = self.subject + '-' + str(self.latest_patchset)
         c = self.add_fake_change_to_repo(msg, fn, large)
         ps_files = [{'file': '/COMMIT_MSG',
@@ -821,8 +822,11 @@ class ZuulTestCase(testtools.TestCase):
                 level=logging.DEBUG,
                 format='%(asctime)s %(name)-32s '
                 '%(levelname)-8s %(message)s'))
-        tmp_root = self.useFixture(fixtures.TempDir(
-            rootdir=os.environ.get("ZUUL_TEST_ROOT"))).path
+        if USE_TEMPDIR:
+            tmp_root = self.useFixture(fixtures.TempDir(
+                    rootdir=os.environ.get("ZUUL_TEST_ROOT"))).path
+        else:
+            tmp_root = os.environ.get("ZUUL_TEST_ROOT")
         self.test_root = os.path.join(tmp_root, "zuul-test")
         self.upstream_root = os.path.join(self.test_root, "upstream")
         self.git_root = os.path.join(self.test_root, "git")
@@ -844,6 +848,7 @@ class ZuulTestCase(testtools.TestCase):
         self.init_repo("org/project1")
         self.init_repo("org/project2")
         self.init_repo("org/project3")
+        self.init_repo("org/project4")
         self.init_repo("org/one-job-project")
         self.init_repo("org/nonvoting-project")
         self.init_repo("org/templated-project")
@@ -998,15 +1003,26 @@ class ZuulTestCase(testtools.TestCase):
         master = repo.create_head('master')
         repo.create_tag('init')
 
-        mp = repo.create_head('mp')
-        repo.head.reference = mp
+        repo.head.reference = master
+        repo.head.reset(index=True, working_tree=True)
+        repo.git.clean('-x', '-f', '-d')
+
+        self.create_branch(project, 'mp')
+
+    def create_branch(self, project, branch):
+        path = os.path.join(self.upstream_root, project)
+        repo = git.Repo.init(path)
+        fn = os.path.join(path, 'README')
+
+        branch_head = repo.create_head(branch)
+        repo.head.reference = branch_head
         f = open(fn, 'a')
-        f.write("test mp\n")
+        f.write("test %s\n" % branch)
         f.close()
         repo.index.add([fn])
-        repo.index.commit('mp commit')
+        repo.index.commit('%s commit' % branch)
 
-        repo.head.reference = master
+        repo.head.reference = repo.heads['master']
         repo.head.reset(index=True, working_tree=True)
         repo.git.clean('-x', '-f', '-d')
 
diff --git a/tests/fixtures/layout-gating.yaml b/tests/fixtures/layout-cloner.yaml
index a544a80f8..258c6c1f9 100644
--- a/tests/fixtures/layout-gating.yaml
+++ b/tests/fixtures/layout-cloner.yaml
@@ -22,8 +22,16 @@ projects:
 
   - name: org/project1
     gate:
-        - project1-project2-integration
+        - integration
 
   - name: org/project2
     gate:
-        - project1-project2-integration
+        - integration
+
+  - name: org/project3
+    gate:
+        - integration
+
+  - name: org/project4
+    gate:
+        - integration
diff --git a/tests/test_cloner.py b/tests/test_cloner.py
index 1a4ce88c1..6f24ff5f7 100644
--- a/tests/test_cloner.py
+++ b/tests/test_cloner.py
@@ -41,13 +41,28 @@ class TestCloner(ZuulTestCase):
         self.workspace_root = os.path.join(self.test_root, 'workspace')
 
         self.config.set('zuul', 'layout_config',
-                        'tests/fixtures/layout-gating.yaml')
+                        'tests/fixtures/layout-cloner.yaml')
         self.sched.reconfigure(self.config)
         self.registerJobs()
 
-    def test_cloner(self):
+    def getWorkspaceRepos(self, projects):
+        repos = {}
+        for project in projects:
+            repos[project] = git.Repo(
+                os.path.join(self.workspace_root, project))
+        return repos
+
+    def getUpstreamRepos(self, projects):
+        repos = {}
+        for project in projects:
+            repos[project] = git.Repo(
+                os.path.join(self.upstream_root, project))
+        return repos
+
+    def test_one_branch(self):
         self.worker.hold_jobs_in_build = True
 
+        projects = ['org/project1', 'org/project2']
         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
         B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
         A.addApproval('CRVW', 2)
@@ -59,34 +74,104 @@ class TestCloner(ZuulTestCase):
 
         self.assertEquals(2, len(self.builds), "Two builds are running")
 
-        for build in self.builds:
+        upstream = self.getUpstreamRepos(projects)
+        states = [
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+             },
+            ]
+
+        for number, build in enumerate(self.builds):
             self.log.debug("Build parameters: %s", build.parameters)
-            change_number = int(build.parameters['ZUUL_CHANGE'])
             cloner = zuul.lib.cloner.Cloner(
                 git_base_url=self.upstream_root,
-                projects=['org/project1', 'org/project2'],
+                projects=projects,
                 workspace=self.workspace_root,
-                zuul_branch='master',
+                zuul_branch=build.parameters['ZUUL_BRANCH'],
                 zuul_ref=build.parameters['ZUUL_REF'],
                 zuul_url=self.git_root,
-                branch='master',
-            )
+                )
             cloner.execute()
-            work_repo1 = git.Repo(os.path.join(self.workspace_root,
-                                               'org/project1'))
-            work_repo2 = git.Repo(os.path.join(self.workspace_root,
-                                               'org/project2'))
-            if change_number >= 1:
-                self.assertEquals(
-                    self.builds[0].parameters['ZUUL_COMMIT'],
-                    str(work_repo1.commit('HEAD')))
-            if change_number >= 2:
-                self.assertEquals(
-                    self.builds[1].parameters['ZUUL_COMMIT'],
-                    str(work_repo2.commit('HEAD')))
-            else:
-                self.assertEquals(str(work_repo2.commit('master')),
-                                  str(work_repo2.commit('HEAD')))
+            work = self.getWorkspaceRepos(projects)
+            state = states[number]
+
+            for project in projects:
+                self.assertEquals(state[project],
+                                  str(work[project].commit('HEAD')),
+                                  'Project %s commit for build %s should '
+                                  'be correct' % (project, number))
+
+            shutil.rmtree(self.workspace_root)
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
+
+    def test_multi_branch(self):
+        self.worker.hold_jobs_in_build = True
+        projects = ['org/project1', 'org/project2',
+                    'org/project3', 'org/project4']
+
+        self.create_branch('org/project2', 'stable/havana')
+        self.create_branch('org/project4', 'stable/havana')
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project2', 'stable/havana', 'B')
+        C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
+        A.addApproval('CRVW', 2)
+        B.addApproval('CRVW', 2)
+        C.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
+
+        self.waitUntilSettled()
+
+        self.assertEquals(3, len(self.builds), "Three builds are running")
+
+        upstream = self.getUpstreamRepos(projects)
+        states = [
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].
+                                 commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].
+                                 commit('stable/havana')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project4': str(upstream['org/project4'].
+                                 commit('master')),
+             },
+            ]
+
+        for number, build in enumerate(self.builds):
+            self.log.debug("Build parameters: %s", build.parameters)
+            cloner = zuul.lib.cloner.Cloner(
+                git_base_url=self.upstream_root,
+                projects=projects,
+                workspace=self.workspace_root,
+                zuul_branch=build.parameters['ZUUL_BRANCH'],
+                zuul_ref=build.parameters['ZUUL_REF'],
+                zuul_url=self.git_root,
+                )
+            cloner.execute()
+            work = self.getWorkspaceRepos(projects)
+            state = states[number]
+
+            for project in projects:
+                self.assertEquals(state[project],
+                                  str(work[project].commit('HEAD')),
+                                  'Project %s commit for build %s should '
+                                  'be correct' % (project, number))
             shutil.rmtree(self.workspace_root)
 
         self.worker.hold_jobs_in_build = False
diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
index 0961eb44d..a38281c64 100644
--- a/zuul/lib/cloner.py
+++ b/zuul/lib/cloner.py
@@ -106,22 +106,22 @@ class Cloner(object):
         # Ensure that we don't have stale remotes around
         repo.prune()
 
-        override_zuul_ref = self.zuul_ref
-        # FIXME should be origin HEAD branch which might not be 'master'
-        fallback_branch = 'master'
-        fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
+        indicated_branch = self.branch or self.zuul_branch
+
+        override_zuul_ref = re.sub(self.zuul_branch, indicated_branch,
                                    self.zuul_ref)
 
-        if self.branch:
-            override_zuul_ref = re.sub(self.zuul_branch, self.branch,
-                                       self.zuul_ref)
-            if repo.hasBranch(self.branch):
-                self.log.debug("upstream repo has branch %s", self.branch)
-                fallback_branch = self.branch
-                fallback_zuul_ref = self.zuul_ref
-            else:
-                self.log.exception("upstream repo is missing branch %s",
-                                   self.branch)
+        if repo.hasBranch(indicated_branch):
+            self.log.debug("upstream repo has branch %s", indicated_branch)
+            fallback_branch = indicated_branch
+        else:
+            self.log.debug("upstream repo is missing branch %s",
+                           self.branch)
+            # FIXME should be origin HEAD branch which might not be 'master'
+            fallback_branch = 'master'
+
+        fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
+                                   self.zuul_ref)
 
         if (self.fetchFromZuul(repo, project, override_zuul_ref)
             or (fallback_zuul_ref != override_zuul_ref and