diff options
54 files changed, 721 insertions, 439 deletions
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst index 8b325baa6..e4ce7379b 100644 --- a/doc/source/zuul.rst +++ b/doc/source/zuul.rst @@ -771,8 +771,11 @@ given pipeline. Within the pipeline section, the jobs that should be executed are listed. If a job is entered as a dictionary key, then jobs contained within that key are only executed if the key job succeeds. In the above example, project-unittest, project-pep8, and -project-pyflakes are only executed if project-merge succeeds. This -can help avoid running unnecessary jobs. +project-pyflakes are only executed if project-merge succeeds. +Furthermore, project-finaltest is executed only if project-unittest, +project-pep8 and project-pyflakes all succeed. This can help avoid +running unnecessary jobs while maximizing parallelism. It is also +useful when distributing results between jobs. The special job named ``noop`` is internal to Zuul and will always return ``SUCCESS`` immediately. This can be useful if you require diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml index aa7005443..50f353d5e 100644 --- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml +++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -17,8 +16,7 @@ name: gate manager: dependent success-message: Build succeeded (gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -41,7 +39,7 @@ pre-run: pre post-run: post vars: - flagpath: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag" + flagpath: '{{zuul._test.test_root}}/{{zuul.uuid}}.flag' roles: - zuul: bare-role diff --git a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml index b38f88e3a..24ba01909 100644 --- a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml +++ b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml @@ -4,7 +4,6 @@ - project: name: org/project - check: jobs: - python27 diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/A.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/A.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/A.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/B.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/B.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/B.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/C.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/C.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/C.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/D.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/D.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/D.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/E.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/E.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/E.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/F.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/F.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/F.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/G.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/G.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/G.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/project-test1.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/project-test1.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml b/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml new file mode 100644 index 000000000..60f365165 --- /dev/null +++ b/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml @@ -0,0 +1,73 @@ +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + source: gerrit + trigger: + gerrit: + - event: comment-added + approval: + - approved: 1 + success: + gerrit: + verified: 2 + submit: true + failure: + gerrit: + verified: -2 + start: + gerrit: + verified: 0 + precedence: high + +- job: + name: A + +- job: + name: B + +- job: + name: C + +- job: + name: D + +- job: + name: E + +- job: + name: F + +- job: + name: G + +- project: + name: org/project + gate: + jobs: + # Job dependencies, starting with A + # A + # / \ + # B C + # / \ / \ + # D F E + # | + # G + # This is intentionally not listed in the natural order to + # ensure that we can reference dependencies before they are + # defined. + - E: + dependencies: C + - A + - B: + dependencies: A + - C: + dependencies: A + - F: + dependencies: + - B + - C + - D: + dependencies: B + - G: + dependencies: F diff --git a/tests/fixtures/config/dependency-graph/git/org_project/README b/tests/fixtures/config/dependency-graph/git/org_project/README new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/tests/fixtures/config/dependency-graph/git/org_project/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/dependency-graph/main.yaml b/tests/fixtures/config/dependency-graph/main.yaml new file mode 100644 index 000000000..d9868fad0 --- /dev/null +++ b/tests/fixtures/config/dependency-graph/main.yaml @@ -0,0 +1,8 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-repos: + - common-config + project-repos: + - org/project diff --git a/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml b/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml index bc88b0617..50051086f 100755 --- a/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml +++ b/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml @@ -2,8 +2,7 @@ name: dup1 manager: independent success-message: Build succeeded (dup1). - source: - gerrit + source: gerrit trigger: gerrit: - event: change-restored @@ -18,8 +17,7 @@ name: dup2 manager: independent success-message: Build succeeded (dup2). - source: - gerrit + source: gerrit trigger: gerrit: - event: change-restored @@ -39,7 +37,6 @@ queue: integrated jobs: - project-test1 - dup2: queue: integrated jobs: diff --git a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml index d8b7200f7..55169ce7b 100644 --- a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml +++ b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -17,8 +16,7 @@ name: tenant-one-gate manager: dependent success-message: Build succeeded (tenant-one-gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added diff --git a/tests/fixtures/config/merges/git/common-config/zuul.yaml b/tests/fixtures/config/merges/git/common-config/zuul.yaml index bb91f3a0c..ab4e24c47 100644 --- a/tests/fixtures/config/merges/git/common-config/zuul.yaml +++ b/tests/fixtures/config/merges/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -17,8 +16,7 @@ name: gate manager: dependent success-message: Build succeeded (gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -37,16 +35,13 @@ precedence: high - job: - name: - project-test1 + name: project-test1 - job: - name: - project-test2 + name: project-test2 - job: - name: - project-merge + name: project-merge hold-following-changes: true - project: @@ -75,6 +70,6 @@ merge-mode: cherry-pick gate: jobs: - - project-merge: - jobs: - - project-test1 + - project-merge + - project-test1: + dependencies: project-merge diff --git a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml index 08117d6d6..004f2df44 100644 --- a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml +++ b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -14,8 +13,7 @@ verified: -1 - job: - name: - python27 + name: python27 nodes: - name: controller image: ubuntu-trusty diff --git a/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml index 4a653f6e4..5769cf543 100644 --- a/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml +++ b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml @@ -2,8 +2,7 @@ name: tenant-one-gate manager: dependent success-message: Build succeeded (tenant-one-gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -28,8 +27,7 @@ image: controller-image - job: - name: - project1-test1 + name: project1-test1 - project: name: org/project1 diff --git a/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml index 7c7972095..19782ce45 100644 --- a/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml +++ b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml @@ -2,8 +2,7 @@ name: tenant-two-gate manager: dependent success-message: Build succeeded (tenant-two-gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -28,8 +27,7 @@ image: controller-image - job: - name: - project2-test1 + name: project2-test1 - project: name: org/project2 diff --git a/tests/fixtures/config/one-job-project/git/common-config/zuul.yaml b/tests/fixtures/config/one-job-project/git/common-config/zuul.yaml index 148ba42c4..4579062ed 100644 --- a/tests/fixtures/config/one-job-project/git/common-config/zuul.yaml +++ b/tests/fixtures/config/one-job-project/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -17,8 +16,7 @@ name: gate manager: dependent success-message: Build succeeded (gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -39,8 +37,7 @@ - pipeline: name: post manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: ref-updated diff --git a/tests/fixtures/config/openstack/git/project-config/zuul.yaml b/tests/fixtures/config/openstack/git/project-config/zuul.yaml index 420d97946..760adb8e8 100644 --- a/tests/fixtures/config/openstack/git/project-config/zuul.yaml +++ b/tests/fixtures/config/openstack/git/project-config/zuul.yaml @@ -1,11 +1,8 @@ -# Pipeline definitions - - pipeline: name: check manager: independent success-message: Build succeeded (check). - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -20,8 +17,7 @@ name: gate manager: dependent success-message: Build succeeded (gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -39,8 +35,6 @@ verified: 0 precedence: high -# Job definitions - - job: name: base timeout: 30 @@ -78,8 +72,6 @@ - openstack/keystone - openstack/nova -# Project definitions - - project: name: openstack/nova templates: diff --git a/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml index 09e0cc6ef..78d2a18a4 100644 --- a/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml +++ b/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: pipeline manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -19,8 +18,7 @@ - pipeline: name: trigger manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added diff --git a/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml index cd76afd5b..1e84e1807 100644 --- a/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml +++ b/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: pipeline manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -20,8 +19,7 @@ - pipeline: name: trigger manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added diff --git a/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml index 8dca5e6e6..efbd79a6b 100644 --- a/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml +++ b/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: pipeline manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -20,8 +19,7 @@ - pipeline: name: trigger manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added diff --git a/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml index 92c7de2b5..7212944e1 100644 --- a/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml +++ b/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml @@ -1,11 +1,10 @@ - pipeline: name: pipeline manager: independent - source: - gerrit + source: gerrit reject: approval: - - username: 'jenkins' + - username: jenkins trigger: gerrit: - event: comment-added @@ -19,13 +18,12 @@ - pipeline: name: trigger manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added reject-approval: - - username: 'jenkins' + - username: jenkins success: gerrit: verified: 1 diff --git a/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml index 12a253829..9f5b12527 100644 --- a/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml +++ b/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml @@ -1,15 +1,18 @@ - pipeline: name: pipeline manager: independent - source: - gerrit + source: gerrit require: approval: - username: jenkins - verified: [1, 2] + verified: + - 1 + - 2 reject: approval: - - verified: [-1, -2] + - verified: + - -1 + - -2 trigger: gerrit: - event: comment-added @@ -23,16 +26,19 @@ - pipeline: name: trigger manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added require-approval: - username: jenkins - verified: [1, 2] + verified: + - 1 + - 2 reject-approval: - - verified: [-1, -2] + - verified: + - -1 + - -2 success: gerrit: verified: 1 diff --git a/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml index 9491bfff7..01ceb46a9 100644 --- a/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml +++ b/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml @@ -1,10 +1,9 @@ - pipeline: name: current-check manager: independent - source: - gerrit + source: gerrit require: - current-patchset: True + current-patchset: true trigger: gerrit: - event: patchset-created @@ -19,10 +18,9 @@ - pipeline: name: open-check manager: independent - source: - gerrit + source: gerrit require: - open: True + open: true trigger: gerrit: - event: patchset-created @@ -37,8 +35,7 @@ - pipeline: name: status-check manager: independent - source: - gerrit + source: gerrit require: status: NEW trigger: diff --git a/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml index ca2ff9792..9789e716e 100644 --- a/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml +++ b/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: pipeline manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -19,8 +18,7 @@ - pipeline: name: trigger manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added diff --git a/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml index 00afe79cf..7989363be 100644 --- a/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml +++ b/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml @@ -1,4 +1,3 @@ - - pipeline: name: pipeline manager: independent @@ -6,8 +5,7 @@ approval: - username: jenkins verified: 1 - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -21,8 +19,7 @@ - pipeline: name: trigger manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added diff --git a/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml index 73db7a73a..9348afbe5 100644 --- a/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml +++ b/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml @@ -4,9 +4,10 @@ require: approval: - username: jenkins - verified: [1, 2] - source: - gerrit + verified: + - 1 + - 2 + source: gerrit trigger: gerrit: - event: comment-added @@ -20,14 +21,15 @@ - pipeline: name: trigger manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added require-approval: - username: jenkins - verified: [1, 2] + verified: + - 1 + - 2 success: gerrit: verified: 1 diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml index b91bf6f4a..47c173dfc 100644 --- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -17,8 +16,7 @@ name: gate manager: dependent success-message: Build succeeded (gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -39,8 +37,7 @@ - pipeline: name: post manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: ref-updated @@ -49,8 +46,7 @@ - pipeline: name: experimental manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -107,23 +103,26 @@ - job: name: project-testfile files: - - '.*-requires' + - .*-requires - project: name: org/project check: jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge gate: jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 - - project-testfile + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project-testfile: + dependencies: project-merge post: jobs: - project-post @@ -132,48 +131,58 @@ name: org/project1 check: jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 - - project1-project2-integration + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge gate: queue: integrated jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 - - project1-project2-integration + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge - project: name: org/project2 gate: queue: integrated jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 - - project1-project2-integration + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge - project: name: org/project3 check: jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 - - project1-project2-integration + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge gate: queue: integrated jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 - - project1-project2-integration + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge post: jobs: - project-post @@ -182,9 +191,9 @@ name: org/experimental-project experimental: jobs: - - project-merge: - jobs: - - experimental-project-test + - project-merge + - experimental-project-test: + dependencies: project-merge - project: name: org/noop-project @@ -199,16 +208,18 @@ name: org/nonvoting-project check: jobs: - - nonvoting-project-merge: - jobs: - - nonvoting-project-test1 - - nonvoting-project-test2 + - nonvoting-project-merge + - nonvoting-project-test1: + dependencies: nonvoting-project-merge + - nonvoting-project-test2: + dependencies: nonvoting-project-merge gate: jobs: - - nonvoting-project-merge: - jobs: - - nonvoting-project-test1 - - nonvoting-project-test2 + - nonvoting-project-merge + - nonvoting-project-test1: + dependencies: nonvoting-project-merge + - nonvoting-project-test2: + dependencies: nonvoting-project-merge - project: name: org/no-jobs-project diff --git a/tests/fixtures/config/single-tenant/git/layout-disabled-at/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-disabled-at/zuul.yaml index 4cf6f1616..bdc19ac63 100644 --- a/tests/fixtures/config/single-tenant/git/layout-disabled-at/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-disabled-at/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created diff --git a/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/zuul.yaml index 30e574ab5..334d9ac88 100644 --- a/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/zuul.yaml @@ -1,13 +1,12 @@ - pipeline: name: post manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: ref-updated ref: ^(?!refs/).*$ - ignore-deletes: False + ignore-deletes: false - job: name: project-post @@ -20,4 +19,3 @@ post: jobs: - project-post - diff --git a/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml index 0c040705b..c69837866 100644 --- a/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml @@ -2,8 +2,7 @@ name: gate manager: dependent success-message: Build succeeded (gate). - source: - gerrit + source: gerrit failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures footer-message: For CI problems and help debugging, contact ci@example.org trigger: @@ -35,4 +34,3 @@ gate: jobs: - project-test1 - diff --git a/tests/fixtures/config/single-tenant/git/layout-idle/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-idle/zuul.yaml index f71f3e4b8..d1fa04bc5 100644 --- a/tests/fixtures/config/single-tenant/git/layout-idle/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-idle/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: periodic manager: independent - source: - gerrit + source: gerrit trigger: timer: - time: '* * * * * */1' diff --git a/tests/fixtures/config/single-tenant/git/layout-inheritance/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-inheritance/zuul.yaml index 3070af066..ab8c9a518 100644 --- a/tests/fixtures/config/single-tenant/git/layout-inheritance/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-inheritance/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -13,7 +12,6 @@ gerrit: verified: -1 - - job: name: project-test-irrelevant-starts-empty diff --git a/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/zuul.yaml index f243bccb1..5d72fc090 100644 --- a/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -13,7 +12,6 @@ gerrit: verified: -1 - - job: name: project-test-irrelevant-files diff --git a/tests/fixtures/config/single-tenant/git/layout-mutex-reconfiguration/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-mutex-reconfiguration/zuul.yaml index 12f174758..0e332e4ba 100644 --- a/tests/fixtures/config/single-tenant/git/layout-mutex-reconfiguration/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-mutex-reconfiguration/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created diff --git a/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml index e91903a59..bb92b7a94 100644 --- a/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created diff --git a/tests/fixtures/config/single-tenant/git/layout-no-timer/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-no-timer/zuul.yaml index f754e376a..ab919a46a 100644 --- a/tests/fixtures/config/single-tenant/git/layout-no-timer/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-no-timer/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -18,8 +17,7 @@ manager: independent # Trigger is required, set it to one that is a noop # during tests that check the timer trigger. - source: - gerrit + source: gerrit trigger: gerrit: - event: ref-updated diff --git a/tests/fixtures/config/single-tenant/git/layout-repo-deleted/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-repo-deleted/zuul.yaml index 2bffc3ed0..5851d7551 100644 --- a/tests/fixtures/config/single-tenant/git/layout-repo-deleted/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-repo-deleted/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -17,8 +16,7 @@ name: gate manager: dependent success-message: Build succeeded (gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -60,13 +58,15 @@ name: org/delete-project check: jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge gate: jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge diff --git a/tests/fixtures/config/single-tenant/git/layout-smtp/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-smtp/zuul.yaml index 9effb1fea..be90d48ae 100644 --- a/tests/fixtures/config/single-tenant/git/layout-smtp/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-smtp/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -23,8 +22,7 @@ name: gate manager: dependent success-message: Build succeeded (gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -69,13 +67,15 @@ name: org/project check: jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge gate: jobs: - - project-merge: - jobs: - - project-test1 - - project-test2 + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge diff --git a/tests/fixtures/config/single-tenant/git/layout-tags/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-tags/zuul.yaml index c921c909f..07f06572b 100644 --- a/tests/fixtures/config/single-tenant/git/layout-tags/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-tags/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -34,19 +33,23 @@ check: jobs: - merge: - jobs: - - test1 - - test2 - - integration tags: - extratag + - test1: + dependencies: merge + - test2: + dependencies: merge + - integration: + dependencies: merge - project: name: org/project2 check: jobs: - - merge: - jobs: - - test1 - - test2 - - integration + - merge + - test1: + dependencies: merge + - test2: + dependencies: merge + - integration: + dependencies: merge diff --git a/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml index 4a14107e7..2a2eca5fe 100644 --- a/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: periodic manager: independent - source: - gerrit + source: gerrit trigger: timer: - time: '* * * * * */1' @@ -10,7 +9,7 @@ smtp: to: alternative_me@example.com from: zuul_from@example.com - subject: 'Periodic check for {change.project} succeeded' + subject: Periodic check for {change.project} succeeded - job: name: project-bitrot-stable-old diff --git a/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml index f69a91da4..80726445f 100644 --- a/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -16,8 +15,7 @@ - pipeline: name: periodic manager: independent - source: - gerrit + source: gerrit trigger: timer: - time: '* * * * * */1' diff --git a/tests/fixtures/config/success-url/git/common-config/zuul.yaml b/tests/fixtures/config/success-url/git/common-config/zuul.yaml index 7edb340d8..f2d5251fa 100644 --- a/tests/fixtures/config/success-url/git/common-config/zuul.yaml +++ b/tests/fixtures/config/success-url/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -18,7 +17,6 @@ gerrit: verified: -1 - - job: name: docs-draft-test success-url: http://docs-draft.example.org/{build.parameters[LOG_PATH]}/publish-docs/ diff --git a/tests/fixtures/config/templated-project/git/common-config/zuul.yaml b/tests/fixtures/config/templated-project/git/common-config/zuul.yaml index 22a2d6d90..8d2c8a047 100644 --- a/tests/fixtures/config/templated-project/git/common-config/zuul.yaml +++ b/tests/fixtures/config/templated-project/git/common-config/zuul.yaml @@ -1,8 +1,7 @@ - pipeline: name: check manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: patchset-created @@ -17,8 +16,7 @@ name: gate manager: dependent success-message: Build succeeded (gate). - source: - gerrit + source: gerrit trigger: gerrit: - event: comment-added @@ -39,8 +37,7 @@ - pipeline: name: post manager: independent - source: - gerrit + source: gerrit trigger: gerrit: - event: ref-updated @@ -56,15 +53,15 @@ - project-template: name: test-three-and-four check: - jobs: - - layered-project-test3 - - layered-project-test4 + jobs: + - layered-project-test3 + - layered-project-test4 - project-template: name: test-five check: jobs: - - layered-project-foo-test5 + - layered-project-foo-test5 - job: name: project-test1 diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 04d147381..ee7c6abe7 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -225,7 +225,7 @@ class TestJob(BaseTestCase): self.assertFalse(python27diablo.changeMatches(change)) self.assertFalse(python27essex.changeMatches(change)) - item.freezeJobTree() + item.freezeJobGraph() self.assertEqual(len(item.getJobs()), 1) job = item.getJobs()[0] self.assertEqual(job.name, 'python27') @@ -253,7 +253,7 @@ class TestJob(BaseTestCase): self.assertTrue(python27diablo.changeMatches(change)) self.assertFalse(python27essex.changeMatches(change)) - item.freezeJobTree() + item.freezeJobGraph() self.assertEqual(len(item.getJobs()), 1) job = item.getJobs()[0] self.assertEqual(job.name, 'python27') @@ -282,7 +282,7 @@ class TestJob(BaseTestCase): self.assertFalse(python27diablo.changeMatches(change)) self.assertTrue(python27essex.changeMatches(change)) - item.freezeJobTree() + item.freezeJobGraph() self.assertEqual(len(item.getJobs()), 1) job = item.getJobs()[0] self.assertEqual(job.name, 'python27') @@ -439,7 +439,7 @@ class TestJob(BaseTestCase): self.assertTrue(python27.changeMatches(change)) self.assertFalse(python27diablo.changeMatches(change)) - item.freezeJobTree() + item.freezeJobGraph() self.assertEqual(len(item.getJobs()), 1) job = item.getJobs()[0] self.assertEqual(job.name, 'python27') @@ -453,7 +453,7 @@ class TestJob(BaseTestCase): self.assertTrue(python27.changeMatches(change)) self.assertTrue(python27diablo.changeMatches(change)) - item.freezeJobTree() + item.freezeJobGraph() self.assertEqual(len(item.getJobs()), 1) job = item.getJobs()[0] self.assertEqual(job.name, 'python27') @@ -506,7 +506,7 @@ class TestJob(BaseTestCase): self.assertTrue(base.changeMatches(change)) self.assertFalse(python27.changeMatches(change)) - item.freezeJobTree() + item.freezeJobGraph() self.assertEqual([], item.getJobs()) def test_job_source_project(self): @@ -609,3 +609,56 @@ class TestTimeDataBase(BaseTestCase): for x in range(10): self.db.update('job-name', 100, 'SUCCESS') self.assertEqual(self.db.getEstimatedTime('job-name'), 100) + + +class TestGraph(BaseTestCase): + def test_job_graph_disallows_multiple_jobs_with_same_name(self): + graph = model.JobGraph() + job1 = model.Job('job') + job2 = model.Job('job') + graph.addJob(job1) + with testtools.ExpectedException(Exception, + "Job job already added"): + graph.addJob(job2) + + def test_job_graph_disallows_circular_dependencies(self): + graph = model.JobGraph() + jobs = [model.Job('job%d' % i) for i in range(0, 10)] + prevjob = None + for j in jobs[:3]: + if prevjob: + j.dependencies = frozenset([prevjob.name]) + graph.addJob(j) + prevjob = j + # 0 triggers 1 triggers 2 triggers 3... + + # Cannot depend on itself + with testtools.ExpectedException( + Exception, + "Dependency cycle detected in job jobX"): + j = model.Job('jobX') + j.dependencies = frozenset([j.name]) + graph.addJob(j) + + # Disallow circular dependencies + with testtools.ExpectedException( + Exception, + "Dependency cycle detected in job job3"): + jobs[4].dependencies = frozenset([jobs[3].name]) + graph.addJob(jobs[4]) + jobs[3].dependencies = frozenset([jobs[4].name]) + graph.addJob(jobs[3]) + + jobs[5].dependencies = frozenset([jobs[4].name]) + graph.addJob(jobs[5]) + + with testtools.ExpectedException( + Exception, + "Dependency cycle detected in job job3"): + jobs[3].dependencies = frozenset([jobs[5].name]) + graph.addJob(jobs[3]) + + jobs[3].dependencies = frozenset([jobs[2].name]) + graph.addJob(jobs[3]) + jobs[6].dependencies = frozenset([jobs[2].name]) + graph.addJob(jobs[6]) diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index e32e41bc0..a923ff15f 100755 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -4478,6 +4478,117 @@ For CI problems and help debugging, contact ci@example.org""" self.assertIn('project-test2 : SKIPPED', A.messages[1]) +class TestDependencyGraph(ZuulTestCase): + tenant_config_file = 'config/dependency-graph/main.yaml' + + def test_dependeny_graph_dispatch_jobs_once(self): + "Test a job in a dependency graph is queued only once" + # Job dependencies, starting with A + # A + # / \ + # B C + # / \ / \ + # D F E + # | + # G + + self.executor_server.hold_jobs_in_build = True + change = self.fake_gerrit.addFakeChange( + 'org/project', 'master', 'change') + change.addApproval('code-review', 2) + self.fake_gerrit.addEvent(change.addApproval('approved', 1)) + + self.waitUntilSettled() + self.assertEqual([b.name for b in self.builds], ['A']) + + self.executor_server.release('A') + self.waitUntilSettled() + self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C']) + + self.executor_server.release('B') + self.waitUntilSettled() + self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D']) + + self.executor_server.release('D') + self.waitUntilSettled() + self.assertEqual([b.name for b in self.builds], ['C']) + + self.executor_server.release('C') + self.waitUntilSettled() + self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F']) + + self.executor_server.release('F') + self.waitUntilSettled() + self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G']) + + self.executor_server.release('G') + self.waitUntilSettled() + self.assertEqual([b.name for b in self.builds], ['E']) + + self.executor_server.release('E') + self.waitUntilSettled() + self.assertEqual(len(self.builds), 0) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 0) + self.assertEqual(len(self.history), 7) + + self.assertEqual(change.data['status'], 'MERGED') + self.assertEqual(change.reported, 2) + + def test_jobs_launched_only_if_all_dependencies_are_successful(self): + "Test that a job waits till all dependencies are successful" + # Job dependencies, starting with A + # A + # / \ + # B C* + # / \ / \ + # D F E + # | + # G + + self.executor_server.hold_jobs_in_build = True + change = self.fake_gerrit.addFakeChange( + 'org/project', 'master', 'change') + change.addApproval('code-review', 2) + + self.executor_server.failJob('C', change) + + self.fake_gerrit.addEvent(change.addApproval('approved', 1)) + + self.waitUntilSettled() + self.assertEqual([b.name for b in self.builds], ['A']) + + self.executor_server.release('A') + self.waitUntilSettled() + self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C']) + + self.executor_server.release('B') + self.waitUntilSettled() + self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D']) + + self.executor_server.release('D') + self.waitUntilSettled() + self.assertEqual([b.name for b in self.builds], ['C']) + + self.executor_server.release('C') + self.waitUntilSettled() + self.assertEqual(len(self.builds), 0) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 0) + self.assertEqual(len(self.history), 4) + + self.assertEqual(change.data['status'], 'NEW') + self.assertEqual(change.reported, 2) + + class TestDuplicatePipeline(ZuulTestCase): tenant_config_file = 'config/duplicate-pipeline/main.yaml' diff --git a/zuul/configloader.py b/zuul/configloader.py index 3f722125b..8bae3c5f2 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -193,6 +193,7 @@ class JobParser(object): 'roles': to_list(role), 'repos': to_list(str), 'vars': dict, + 'dependencies': to_list(str), } return vs.Schema(job) @@ -276,6 +277,8 @@ class JobParser(object): # accumulate onto any previously applied tags. job.tags = job.tags.union(set(tags)) + job.dependencies = frozenset(as_list(conf.get('dependencies'))) + roles = [] for role in conf.get('roles', []): if 'zuul' in role: @@ -364,45 +367,33 @@ class ProjectTemplateParser(object): project_pipeline = model.ProjectPipelineConfig() project_template.pipelines[pipeline.name] = project_pipeline project_pipeline.queue_name = conf_pipeline.get('queue') - project_pipeline.job_tree = ProjectTemplateParser._parseJobTree( + ProjectTemplateParser._parseJobList( tenant, layout, conf_pipeline.get('jobs', []), - source_context, start_mark) + source_context, start_mark, project_pipeline.job_list) return project_template @staticmethod - def _parseJobTree(tenant, layout, conf, source_context, - start_mark, tree=None): - if not tree: - tree = model.JobTree(None) + def _parseJobList(tenant, layout, conf, source_context, + start_mark, job_list): for conf_job in conf: if isinstance(conf_job, six.string_types): job = model.Job(conf_job) - tree.addJob(job) + job_list.addJob(job) elif isinstance(conf_job, dict): - # A dictionary in a job tree may override params, or - # be the root of a sub job tree, or both. + # A dictionary in a job tree may override params jobname, attrs = conf_job.items()[0] - jobs = attrs.pop('jobs', None) if attrs: # We are overriding params, so make a new job def attrs['name'] = jobname attrs['_source_context'] = source_context attrs['_start_mark'] = start_mark - subtree = tree.addJob(JobParser.fromYaml( - tenant, layout, attrs)) + job_list.addJob(JobParser.fromYaml(tenant, layout, attrs)) else: # Not overriding, so add a blank job job = model.Job(jobname) - subtree = tree.addJob(job) - - if jobs: - # This is the root of a sub tree - ProjectTemplateParser._parseJobTree( - tenant, layout, jobs, source_context, - start_mark, subtree) + job_list.addJob(job) else: raise Exception("Job must be a string or dictionary") - return tree class ProjectParser(object): @@ -455,7 +446,6 @@ class ProjectParser(object): project.merge_mode = model.MERGER_MAP['merge-resolve'] for pipeline in layout.pipelines.values(): project_pipeline = model.ProjectPipelineConfig() - project_pipeline.job_tree = model.JobTree(None) queue_name = None # For every template, iterate over the job tree and replace or # create the jobs in the final definition as needed. @@ -467,8 +457,8 @@ class ProjectParser(object): (template.name, pipeline.name)) pipeline_defined = True template_pipeline = template.pipelines[pipeline.name] - project_pipeline.job_tree.inheritFrom( - template_pipeline.job_tree) + project_pipeline.job_list.inheritFrom( + template_pipeline.job_list) if template_pipeline.queue_name: queue_name = template_pipeline.queue_name if queue_name: diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py index e7508b688..58ad6077c 100644 --- a/zuul/manager/__init__.py +++ b/zuul/manager/__init__.py @@ -64,31 +64,29 @@ class PipelineManager(object): self.log.info(" %s" % e) self.log.info(" Projects:") - def log_jobs(tree, indent=0): - istr = ' ' + ' ' * indent - if tree.job: - # TODOv3(jeblair): represent matchers - efilters = '' - # for b in tree.job._branches: - # efilters += str(b) - # for f in tree.job._files: - # efilters += str(f) - # if tree.job.skip_if_matcher: - # efilters += str(tree.job.skip_if_matcher) - # if efilters: - # efilters = ' ' + efilters - tags = [] - if tree.job.hold_following_changes: - tags.append('[hold]') - if not tree.job.voting: - tags.append('[nonvoting]') - if tree.job.mutex: - tags.append('[mutex: %s]' % tree.job.mutex) - tags = ' '.join(tags) - self.log.info("%s%s%s %s" % (istr, repr(tree.job), - efilters, tags)) - for x in tree.job_trees: - log_jobs(x, indent + 2) + def log_jobs(job_list): + for job_name, job_variants in job_list.jobs.items(): + for variant in job_variants: + # TODOv3(jeblair): represent matchers + efilters = '' + # for b in tree.job._branches: + # efilters += str(b) + # for f in tree.job._files: + # efilters += str(f) + # if tree.job.skip_if_matcher: + # efilters += str(tree.job.skip_if_matcher) + # if efilters: + # efilters = ' ' + efilters + tags = [] + if variant.hold_following_changes: + tags.append('[hold]') + if not variant.voting: + tags.append('[nonvoting]') + if variant.mutex: + tags.append('[mutex: %s]' % variant.mutex) + tags = ' '.join(tags) + self.log.info(" %s%s %s" % (repr(variant), + efilters, tags)) for project_name in layout.project_configs.keys(): project_config = layout.project_configs.get(project_name) @@ -97,7 +95,7 @@ class PipelineManager(object): self.pipeline.name) if project_pipeline_config: self.log.info(" %s" % project_name) - log_jobs(project_pipeline_config.job_tree) + log_jobs(project_pipeline_config.job_list) self.log.info(" On start:") self.log.info(" %s" % self.pipeline.start_actions) self.log.info(" On success:") @@ -257,7 +255,7 @@ class PipelineManager(object): # Rebuild the frozen job tree from the new layout, if # we have one. If not, it will be built later. if item.current_build_set.layout: - item.freezeJobTree() + item.freezeJobGraph() # Re-set build results in case any new jobs have been # added to the tree. @@ -540,8 +538,18 @@ class PipelineManager(object): item.current_build_set.layout = self.getLayout(item) if not item.current_build_set.layout: return False - if not item.job_tree: - item.freezeJobTree() + if item.current_build_set.config_error: + return False + if not item.job_graph: + try: + item.freezeJobGraph() + except Exception as e: + # TODOv3(jeblair): nicify this exception as it will be reported + self.log.exception("Error freezing job graph for %s" % + item) + item.setConfigError("Unable to freeze job graph: %s" % + (str(e))) + return False return True def _processOneItem(self, item, nnfi): diff --git a/zuul/model.py b/zuul/model.py index b57eef874..3676b687a 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -675,6 +675,7 @@ class Job(object): file_matcher=None, irrelevant_file_matcher=None, # skip-if tags=frozenset(), + dependencies=frozenset(), ) # These attributes affect how the job is actually run and more @@ -851,60 +852,100 @@ class Job(object): return True -class JobTree(object): - """A JobTree holds one or more Jobs to represent Job dependencies. +class JobList(object): + """ A list of jobs in a project's pipeline. """ - If Job foo should only execute if Job bar succeeds, then there will - be a JobTree for foo, which will contain a JobTree for bar. A JobTree - can hold more than one dependent JobTrees, such that jobs bar and bang - both depend on job foo being successful. + def __init__(self): + self.jobs = OrderedDict() # job.name -> [job, ...] - A root node of a JobTree will have no associated Job.""" + def addJob(self, job): + if job.name in self.jobs: + self.jobs[job.name].append(job) + else: + self.jobs[job.name] = [job] - def __init__(self, job): - self.job = job - self.job_trees = [] + def inheritFrom(self, other): + for jobname, jobs in other.jobs.items(): + if jobname in self.jobs: + self.jobs[jobname].append(jobs) + else: + self.jobs[jobname] = jobs + + +class JobGraph(object): + """ A JobGraph represents the dependency graph between Job.""" + + def __init__(self): + self.jobs = OrderedDict() # job_name -> Job + self._dependencies = {} # dependent_job_name -> set(parent_job_names) def __repr__(self): - return '<JobTree %s %s>' % (self.job, self.job_trees) + return '<JobGraph %s>' % (self.jobs) def addJob(self, job): - if job not in [x.job for x in self.job_trees]: - t = JobTree(job) - self.job_trees.append(t) - return t - for tree in self.job_trees: - if tree.job == job: - return tree + # A graph must be created after the job list is frozen, + # therefore we should only get one job with the same name. + if job.name in self.jobs: + raise Exception("Job %s already added" % (job.name,)) + self.jobs[job.name] = job + # Append the dependency information + self._dependencies.setdefault(job.name, set()) + try: + for dependency in job.dependencies: + # Make sure a circular dependency is never created + ancestor_jobs = self._getParentJobNamesRecursively( + dependency, soft=True) + ancestor_jobs.add(dependency) + if any((job.name == anc_job) for anc_job in ancestor_jobs): + raise Exception("Dependency cycle detected in job %s" % + (job.name,)) + self._dependencies[job.name].add(dependency) + except Exception: + del self.jobs[job.name] + del self._dependencies[job.name] + raise def getJobs(self): - jobs = [] - for x in self.job_trees: - jobs.append(x.job) - jobs.extend(x.getJobs()) - return jobs - - def getJobTreeForJob(self, job): - if self.job == job: - return self - for tree in self.job_trees: - ret = tree.getJobTreeForJob(job) - if ret: - return ret - return None + return self.jobs.values() # Report in the order of the layout config - def inheritFrom(self, other): - if other.job: - if not self.job: - self.job = other.job.copy() - else: - self.job.applyVariant(other.job) - for other_tree in other.job_trees: - this_tree = self.getJobTreeForJob(other_tree.job) - if not this_tree: - this_tree = JobTree(None) - self.job_trees.append(this_tree) - this_tree.inheritFrom(other_tree) + def _getDirectDependentJobs(self, parent_job): + ret = set() + for dependent_name, parent_names in self._dependencies.items(): + if parent_job in parent_names: + ret.add(dependent_name) + return ret + + def getDependentJobsRecursively(self, parent_job): + all_dependent_jobs = set() + jobs_to_iterate = set([parent_job]) + while len(jobs_to_iterate) > 0: + current_job = jobs_to_iterate.pop() + current_dependent_jobs = self._getDirectDependentJobs(current_job) + new_dependent_jobs = current_dependent_jobs - all_dependent_jobs + jobs_to_iterate |= new_dependent_jobs + all_dependent_jobs |= new_dependent_jobs + return [self.jobs[name] for name in all_dependent_jobs] + + def getParentJobsRecursively(self, dependent_job): + return [self.jobs[name] for name in + self._getParentJobNamesRecursively(dependent_job)] + + def _getParentJobNamesRecursively(self, dependent_job, soft=False): + all_parent_jobs = set() + jobs_to_iterate = set([dependent_job]) + while len(jobs_to_iterate) > 0: + current_job = jobs_to_iterate.pop() + current_parent_jobs = self._dependencies.get(current_job) + if current_parent_jobs is None: + if soft: + current_parent_jobs = set() + else: + raise Exception("Dependent job %s not found: " % + (dependent_job,)) + new_parent_jobs = current_parent_jobs - all_parent_jobs + jobs_to_iterate |= new_parent_jobs + all_parent_jobs |= new_parent_jobs + return all_parent_jobs class Build(object): @@ -1137,7 +1178,7 @@ class QueueItem(object): self.active = False # Whether an item is within an active window self.live = True # Whether an item is intended to be processed at all self.layout = None # This item's shadow layout - self.job_tree = None + self.job_graph = None def __repr__(self): if self.pipeline: @@ -1165,23 +1206,33 @@ class QueueItem(object): def setReportedResult(self, result): self.current_build_set.result = result - def freezeJobTree(self): + def freezeJobGraph(self): """Find or create actual matching jobs for this item's change and store the resulting job tree.""" layout = self.current_build_set.layout - self.job_tree = layout.createJobTree(self) + job_graph = layout.createJobGraph(self) + for job in job_graph.getJobs(): + # Ensure that each jobs's dependencies are fully + # accessible. This will raise an exception if not. + job_graph.getParentJobsRecursively(job.name) + self.job_graph = job_graph - def hasJobTree(self): - """Returns True if the item has a job tree.""" - return self.job_tree is not None + def hasJobGraph(self): + """Returns True if the item has a job graph.""" + return self.job_graph is not None def getJobs(self): - if not self.live or not self.job_tree: + if not self.live or not self.job_graph: return [] - return self.job_tree.getJobs() + return self.job_graph.getJobs() + + def getJob(self, name): + if not self.job_graph: + return None + return self.job_graph.jobs.get(name) def haveAllJobsStarted(self): - if not self.hasJobTree(): + if not self.hasJobGraph(): return False for job in self.getJobs(): build = self.current_build_set.getBuild(job.name) @@ -1193,7 +1244,7 @@ class QueueItem(object): if (self.current_build_set.config_error or self.current_build_set.unable_to_merge): return True - if not self.hasJobTree(): + if not self.hasJobGraph(): return False for job in self.getJobs(): build = self.current_build_set.getBuild(job.name) @@ -1202,7 +1253,7 @@ class QueueItem(object): return True def didAllJobsSucceed(self): - if not self.hasJobTree(): + if not self.hasJobGraph(): return False for job in self.getJobs(): if not job.voting: @@ -1215,7 +1266,7 @@ class QueueItem(object): return True def didAnyJobFail(self): - if not self.hasJobTree(): + if not self.hasJobGraph(): return False for job in self.getJobs(): if not job.voting: @@ -1234,7 +1285,7 @@ class QueueItem(object): def isHoldingFollowingChanges(self): if not self.live: return False - if not self.hasJobTree(): + if not self.hasJobGraph(): return False for job in self.getJobs(): if not job.hold_following_changes: @@ -1249,88 +1300,96 @@ class QueueItem(object): return False return self.item_ahead.isHoldingFollowingChanges() - def _findJobsToRun(self, job_trees, mutex): + def findJobsToRun(self, mutex): torun = [] + if not self.live: + return [] + if not self.job_graph: + return [] if self.item_ahead: # Only run jobs if any 'hold' jobs on the change ahead # have completed successfully. if self.item_ahead.isHoldingFollowingChanges(): return [] - for tree in job_trees: - job = tree.job - result = None - if job: - if not job.changeMatches(self.change): + + successful_job_names = set() + jobs_not_started = set() + for job in self.job_graph.getJobs(): + build = self.current_build_set.getBuild(job.name) + if build: + if build.result == 'SUCCESS': + successful_job_names.add(job.name) + else: + jobs_not_started.add(job) + + # Attempt to request nodes for jobs in the order jobs appear + # in configuration. + for job in self.job_graph.getJobs(): + if job not in jobs_not_started: + continue + all_parent_jobs_successful = True + for parent_job in self.job_graph.getParentJobsRecursively( + job.name): + if parent_job.name not in successful_job_names: + all_parent_jobs_successful = False + break + if all_parent_jobs_successful: + nodeset = self.current_build_set.getJobNodeSet(job.name) + if nodeset is None: + # The nodes for this job are not ready, skip + # it for now. continue - build = self.current_build_set.getBuild(job.name) - if build: - result = build.result - else: - # There is no build for the root of this job tree, - # so it has not run yet. - nodeset = self.current_build_set.getJobNodeSet(job.name) - if nodeset is None: - # The nodes for this job are not ready, skip - # it for now. - continue - if mutex.acquire(self, job): - # If this job needs a mutex, either acquire it or make - # sure that we have it before running the job. - torun.append(job) - # If there is no job, this is a null job tree, and we should - # run all of its jobs. - if result == 'SUCCESS' or not job: - torun.extend(self._findJobsToRun(tree.job_trees, mutex)) + if mutex.acquire(self, job): + # If this job needs a mutex, either acquire it or make + # sure that we have it before running the job. + torun.append(job) return torun - def findJobsToRun(self, mutex): + def findJobsToRequest(self): + build_set = self.current_build_set + toreq = [] if not self.live: return [] - tree = self.job_tree - if not tree: + if not self.job_graph: return [] - return self._findJobsToRun(tree.job_trees, mutex) - - def _findJobsToRequest(self, job_trees): - build_set = self.current_build_set - toreq = [] if self.item_ahead: if self.item_ahead.isHoldingFollowingChanges(): return [] - for tree in job_trees: - job = tree.job - result = None - if job: - if not job.changeMatches(self.change): - continue - build = build_set.getBuild(job.name) - if build: - result = build.result - else: - nodeset = build_set.getJobNodeSet(job.name) - if nodeset is None: - req = build_set.getJobNodeRequest(job.name) - if req is None: - toreq.append(job) - if result == 'SUCCESS' or not job: - toreq.extend(self._findJobsToRequest(tree.job_trees)) - return toreq - def findJobsToRequest(self): - if not self.live: - return [] - tree = self.job_tree - if not tree: - return [] - return self._findJobsToRequest(tree.job_trees) + successful_job_names = set() + jobs_not_requested = set() + for job in self.job_graph.getJobs(): + build = build_set.getBuild(job.name) + if build and build.result == 'SUCCESS': + successful_job_names.add(job.name) + else: + nodeset = build_set.getJobNodeSet(job.name) + if nodeset is None: + req = build_set.getJobNodeRequest(job.name) + if req is None: + jobs_not_requested.add(job) + + # Attempt to request nodes for jobs in the order jobs appear + # in configuration. + for job in self.job_graph.getJobs(): + if job not in jobs_not_requested: + continue + all_parent_jobs_successful = True + for parent_job in self.job_graph.getParentJobsRecursively( + job.name): + if parent_job.name not in successful_job_names: + all_parent_jobs_successful = False + break + if all_parent_jobs_successful: + toreq.append(job) + return toreq def setResult(self, build): if build.retry: self.removeBuild(build) elif build.result != 'SUCCESS': - # Get a JobTree from a Job so we can find only its dependent jobs - tree = self.job_tree.getJobTreeForJob(build.job) - for job in tree.getJobs(): + for job in self.job_graph.getDependentJobsRecursively( + build.job.name): fakebuild = Build(job, None) fakebuild.result = 'SKIPPED' self.addBuild(fakebuild) @@ -2014,7 +2073,7 @@ class ChangeishFilter(BaseFilter): class ProjectPipelineConfig(object): # Represents a project cofiguration in the context of a pipeline def __init__(self): - self.job_tree = None + self.job_list = JobList() self.queue_name = None self.merge_mode = None @@ -2182,14 +2241,13 @@ class Layout(object): def addProjectConfig(self, project_config): self.project_configs[project_config.name] = project_config - def _createJobTree(self, change, job_trees, parent): - for tree in job_trees: - job = tree.job - if not job.changeMatches(change): - continue + def _createJobGraph(self, change, job_list, job_graph): + for jobname in job_list.jobs: + # This is the final job we are constructing frozen_job = None + # Whether the change matches any globally defined variant matched = False - for variant in self.getJobs(job.name): + for variant in self.getJobs(jobname): if variant.changeMatches(change): if frozen_job is None: frozen_job = variant.copy() @@ -2203,25 +2261,33 @@ class Layout(object): # the job that is defined in the tree). continue # If the job does not allow auth inheritance, do not allow - # the project-pipeline variant to update its execution + # the project-pipeline variants to update its execution # attributes. if frozen_job.auth and not frozen_job.auth.get('inherit'): frozen_job.final = True - frozen_job.applyVariant(job) - frozen_tree = JobTree(frozen_job) - parent.job_trees.append(frozen_tree) - self._createJobTree(change, tree.job_trees, frozen_tree) + # Whether the change matches any of the project pipeline + # variants + matched = False + for variant in job_list.jobs[jobname]: + if variant.changeMatches(change): + frozen_job.applyVariant(variant) + matched = True + if not matched: + # A change must match at least one project pipeline + # job variant. + continue + job_graph.addJob(frozen_job) - def createJobTree(self, item): + def createJobGraph(self, item): project_config = self.project_configs.get( item.change.project.name, None) - ret = JobTree(None) + ret = JobGraph() # NOTE(pabelanger): It is possible for a foreign project not to have a - # configured pipeline, if so return an empty JobTree. + # configured pipeline, if so return an empty JobGraph. if project_config and item.pipeline.name in project_config.pipelines: - project_tree = \ - project_config.pipelines[item.pipeline.name].job_tree - self._createJobTree(item.change, project_tree.job_trees, ret) + project_job_list = \ + project_config.pipelines[item.pipeline.name].job_list + self._createJobGraph(item.change, project_job_list, ret) return ret diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 131378e7a..7fb156817 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -555,11 +555,10 @@ class Scheduler(threading.Thread): project_name) if new_pipeline.manager.reEnqueueItem(item, last_head): - new_jobs = item.getJobs() for build in item.current_build_set.getBuilds(): - jobtree = item.job_tree.getJobTreeForJob(build.job) - if jobtree and jobtree.job in new_jobs: - build.job = jobtree.job + new_job = item.getJob(build.job.name) + if new_job: + build.job = new_job else: item.removeBuild(build) builds_to_cancel.append(build) |