Merge "Add nodeset alternatives"

6 files changed, 307 insertions, 1 deletions

diff --git a/tests/fixtures/layouts/nodeset-alternatives.yaml b/tests/fixtures/layouts/nodeset-alternatives.yaml
new file mode 100644
index 000000000..21f9f11ae
--- /dev/null
+++ b/tests/fixtures/layouts/nodeset-alternatives.yaml
@@ -0,0 +1,14 @@
+- nodeset:
+    name: fast-nodeset
+    nodes:
+      - name: controller
+        label: fast-label
+
+- job:
+    name: test-job
+    nodeset:
+      alternatives:
+        - fast-nodeset
+        - nodes:
+            - name: controller
+              label: slow-label
diff --git a/tests/fixtures/layouts/nodeset-fallback.yaml b/tests/fixtures/layouts/nodeset-fallback.yaml
new file mode 100644
index 000000000..01869537e
--- /dev/null
+++ b/tests/fixtures/layouts/nodeset-fallback.yaml
@@ -0,0 +1,66 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- nodeset:
+    name: fast-nodeset
+    nodes:
+      - label: fast-label
+        name: controller
+
+- nodeset:
+    name: red-nodeset
+    nodes:
+      - label: red-label
+        name: controller
+
+- nodeset:
+    name: blue-nodeset
+    nodes:
+      - label: blue-label
+        name: controller
+
+# This adds an unused second level of alternatives to verify we are
+# able to flatten it.
+- nodeset:
+    name: red-or-blue-nodeset
+    alternatives:
+      - red-nodeset
+      - blue-nodeset
+
+# Test alternatives by name or anonymous nodeset
+- nodeset:
+    name: fast-or-slow
+    alternatives:
+      - fast-nodeset
+      - nodes:
+          label: slow-label
+          name: controller
+      - red-or-blue-nodeset
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- job:
+    name: check-job
+    nodeset: fast-or-slow
+
+- project:
+    name: org/project
+    check:
+      jobs:
+        - check-job
+    gate:
+      jobs:
+        - check-job
diff --git a/tests/unit/test_model_upgrade.py b/tests/unit/test_model_upgrade.py
index 2004b317b..020045859 100644
--- a/tests/unit/test_model_upgrade.py
+++ b/tests/unit/test_model_upgrade.py
@@ -215,6 +215,44 @@ class TestModelUpgrade(ZuulTestCase):
     # code paths are exercised in existing tests since small secrets
     # don't use the blob store.
 
+    @model_version(8)
+    def test_model_8_9(self):
+        # This excercises the upgrade to nodeset_alternates
+        first = self.scheds.first
+        second = self.createScheduler()
+        second.start()
+        self.assertEqual(len(self.scheds), 2)
+        for _ in iterate_timeout(10, "until priming is complete"):
+            state_one = first.sched.local_layout_state.get("tenant-one")
+            if state_one:
+                break
+
+        for _ in iterate_timeout(
+                10, "all schedulers to have the same layout state"):
+            if (second.sched.local_layout_state.get(
+                    "tenant-one") == state_one):
+                break
+
+        self.fake_nodepool.pause()
+        with second.sched.layout_update_lock, second.sched.run_handler_lock:
+            A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+            self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+            self.waitUntilSettled(matcher=[first])
+
+        self.model_test_component_info.model_api = 9
+        with first.sched.layout_update_lock, first.sched.run_handler_lock:
+            self.fake_nodepool.unpause()
+            self.waitUntilSettled(matcher=[second])
+
+        self.waitUntilSettled()
+        self.assertHistory([
+            dict(name='project-merge', result='SUCCESS', changes='1,1'),
+            dict(name='project-test1', result='SUCCESS', changes='1,1'),
+            dict(name='project-test2', result='SUCCESS', changes='1,1'),
+            dict(name='project1-project2-integration',
+                 result='SUCCESS', changes='1,1'),
+        ], ordered=False)
+
 
 class TestSemaphoreModelUpgrade(ZuulTestCase):
     tenant_config_file = 'config/semaphore/main.yaml'
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 3445e9dc6..5d8099686 100644
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -36,6 +36,7 @@ from zuul.driver.gerrit import gerritreporter
 import zuul.scheduler
 import zuul.model
 import zuul.merger.merger
+from zuul.lib import yamlutil as yaml
 
 from tests.base import (
     SSLZuulTestCase,
@@ -6089,6 +6090,50 @@ For CI problems and help debugging, contact ci@example.org"""
             self.assertFalse(node['_lock'])
             self.assertEqual(node['state'], 'ready')
 
+    @simple_layout('layouts/nodeset-fallback.yaml')
+    def test_nodeset_fallback(self):
+        # Test that nodeset fallback works
+        self.executor_server.hold_jobs_in_build = True
+
+        # Verify that we get the correct number and order of
+        # alternates from our nested config.
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        job = tenant.layout.getJob('check-job')
+        alts = job.flattenNodesetAlternatives(tenant.layout)
+        self.assertEqual(4, len(alts))
+        self.assertEqual('fast-nodeset', alts[0].name)
+        self.assertEqual('', alts[1].name)
+        self.assertEqual('red-nodeset', alts[2].name)
+        self.assertEqual('blue-nodeset', alts[3].name)
+
+        self.fake_nodepool.pause()
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        req = self.fake_nodepool.getNodeRequests()[0]
+        self.fake_nodepool.addFailRequest(req)
+
+        self.fake_nodepool.unpause()
+        self.waitUntilSettled()
+
+        build = self.getBuildByName('check-job')
+        inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml')
+        inventory = yaml.safe_load(open(inv_path, 'r'))
+        label = inventory['all']['hosts']['controller']['nodepool']['label']
+        self.assertEqual('slow-label', label)
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(A.data['status'], 'NEW')
+        self.assertEqual(A.reported, 1)
+        self.assertNotIn('NODE_FAILURE', A.messages[0])
+        self.assertHistory([
+            dict(name='check-job', result='SUCCESS', changes='1,1'),
+        ], ordered=False)
+
     @simple_layout('layouts/multiple-templates.yaml')
     def test_multiple_project_templates(self):
         # Test that applying multiple project templates to a project
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index d35b23285..3d41d74c6 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -2682,6 +2682,69 @@ class TestInRepoConfig(ZuulTestCase):
         self.assertIn('Debug information:',
                       A.messages[0], "A should have debug info")
 
+    def test_nodeset_alternates_cycle(self):
+        in_repo_conf = textwrap.dedent(
+            """
+            - nodeset:
+                name: red
+                alternatives: [blue]
+            - nodeset:
+                name: blue
+                alternatives: [red]
+            - job:
+                name: project-test1
+                run: playbooks/project-test1.yaml
+                nodeset: blue
+            """)
+
+        file_dict = {'.zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(A.reported, 1)
+        self.assertIn("cycle detected", A.messages[0])
+
+    def test_nodeset_alternates_missing_from_nodeset(self):
+        in_repo_conf = textwrap.dedent(
+            """
+            - nodeset:
+                name: red
+                alternatives: [blue]
+            - job:
+                name: project-test1
+                run: playbooks/project-test1.yaml
+            """)
+
+        file_dict = {'.zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(A.reported, 1)
+        self.assertIn('nodeset "blue" was not found', A.messages[0])
+
+    def test_nodeset_alternates_missing_from_job(self):
+        in_repo_conf = textwrap.dedent(
+            """
+            - job:
+                name: project-test1
+                run: playbooks/project-test1.yaml
+                nodeset:
+                  alternatives: [red]
+            """)
+
+        file_dict = {'.zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(A.reported, 1)
+        self.assertIn('nodeset "red" was not found', A.messages[0])
+
     @skipIfMultiScheduler()
     # See comment in TestInRepoConfigDir.scheduler_count for further
     # details.
diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py
index 616b776b2..f9de11a9b 100644
--- a/tests/unit/test_web.py
+++ b/tests/unit/test_web.py
@@ -341,6 +341,82 @@ class TestWeb(BaseTestWeb):
         self.assertEqual(1, len(data), data)
         self.assertEqual("org/project1", data[0]['project'], data)
 
+    @simple_layout('layouts/nodeset-alternatives.yaml')
+    def test_web_find_job_nodeset_alternatives(self):
+        # test a complex nodeset
+        data = self.get_url('api/tenant/tenant-one/job/test-job').json()
+
+        self.assertEqual([
+            {'abstract': False,
+             'ansible_version': None,
+             'attempts': 3,
+             'branches': [],
+             'cleanup_run': [],
+             'deduplicate': 'auto',
+             'dependencies': [],
+             'description': None,
+             'extra_variables': {},
+             'files': [],
+             'final': False,
+             'group_variables': {},
+             'host_variables': {},
+             'intermediate': False,
+             'irrelevant_files': [],
+             'match_on_config_updates': True,
+             'name': 'test-job',
+             'nodeset_alternatives': [{'alternatives': [],
+                                       'groups': [],
+                                       'name': 'fast-nodeset',
+                                       'nodes': [{'aliases': [],
+                                                  'comment': None,
+                                                  'hold_job': None,
+                                                  'id': None,
+                                                  'label': 'fast-label',
+                                                  'name': 'controller',
+                                                  'requestor': None,
+                                                  'state': 'unknown',
+                                                  'tenant_name': None,
+                                                  'user_data': None}]},
+                                      {'alternatives': [],
+                                       'groups': [],
+                                       'name': '',
+                                       'nodes': [{'aliases': [],
+                                                  'comment': None,
+                                                  'hold_job': None,
+                                                  'id': None,
+                                                  'label': 'slow-label',
+                                                  'name': 'controller',
+                                                  'requestor': None,
+                                                  'state': 'unknown',
+                                                  'tenant_name': None,
+                                                  'user_data': None}]}],
+             'override_checkout': None,
+             'parent': 'base',
+             'post_review': None,
+             'post_run': [],
+             'pre_run': [],
+             'protected': None,
+             'provides': [],
+             'required_projects': [],
+             'requires': [],
+             'roles': [{'implicit': True,
+                        'project_canonical_name':
+                        'review.example.com/org/common-config',
+                        'target_name': 'common-config',
+                        'type': 'zuul'}],
+             'run': [],
+             'semaphores': [],
+             'source_context': {'branch': 'master',
+                                'path': 'zuul.yaml',
+                                'project': 'org/common-config'},
+             'tags': [],
+             'timeout': None,
+             'variables': {},
+             'variant_description': '',
+             'voting': True,
+             'workspace_scheme': 'golang',
+            }], data)
+
     def test_web_find_job(self):
         # can we fetch the variants for a single job
         data = self.get_url('api/tenant/tenant-one/job/project-test1').json()
@@ -384,6 +460,7 @@ class TestWeb(BaseTestWeb):
                 'match_on_config_updates': True,
                 'final': False,
                 'nodeset': {
+                    'alternatives': [],
                     'groups': [],
                     'name': '',
                     'nodes': [{'comment': None,
@@ -435,6 +512,7 @@ class TestWeb(BaseTestWeb):
                 'match_on_config_updates': True,
                 'final': False,
                 'nodeset': {
+                    'alternatives': [],
                     'groups': [],
                     'name': '',
                     'nodes': [{'comment': None,
@@ -1071,6 +1149,7 @@ class TestWeb(BaseTestWeb):
             'branch': 'master',
             'cleanup_playbooks': [],
             'nodeset': {
+                'alternatives': [],
                 'groups': [],
                 'name': '',
                 'nodes': [
@@ -1168,7 +1247,8 @@ class TestWeb(BaseTestWeb):
             'host_vars': {},
             'items': [],
             'job': 'noop',
-            'nodeset': {'groups': [], 'name': '', 'nodes': []},
+            'nodeset': {'alternatives': [],
+                        'groups': [], 'name': '', 'nodes': []},
             'override_branch': None,
             'override_checkout': None,
             'post_timeout': None,