Merge remote-tracking branch 'origin/master' into stream-timestamp-offsetstream-timestamp-offset

9 files changed, 556 insertions, 0 deletions

diff --git a/workflow_sources/test/ct.lib.yml b/workflow_sources/test/ct.lib.yml
new file mode 100644
index 0000000000..5a3666776d
--- /dev/null
+++ b/workflow_sources/test/ct.lib.yml
@@ -0,0 +1,133 @@
+#@ load("@ytt:data", "data")
+#@ load("@ytt:assert", "assert")
+#@ load("util.star", "is_unique", "group_by_one", "to_build_args")
+#@ load("helpers.star", "ci_image_tag", "ci_dep_image", "skip_ci_condition")
+
+#@ def checks_job(dep):
+name: #@ dep.name + "-checks"
+needs: [prepare]
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+outputs:
+  step_start: ${{ steps.buildevents.outputs.step_start }}
+#@yaml/text-templated-strings
+steps:
+  - name: RECORD STEP START
+    id: buildevents
+    run: |
+      echo "::set-output name=step_start::$(date +%s)"
+  - name: CHECKOUT REPOSITORY
+    uses: actions/checkout@v2
+  - name: VALIDATE KNOWN CT SUITES
+    env:
+      project: #@ dep.name
+    #@ suite_names = [suite.name for suite in dep.suites]
+    #@ None if is_unique(suite_names) else assert.fail('{} suite names are not unique'.format(dep.name))
+    run: |
+      ci/scripts/validate-workflow.sh (@= " ".join(suite_names) @)
+  - name: RUN CHECKS
+    uses: docker/build-push-action@v1
+    with:
+      username: _json_key
+      password: ${{ secrets.GCR_JSON_KEY }}
+      registry: eu.gcr.io
+      repository: cf-rabbitmq-core/ci-(@= dep.name @)
+      dockerfile: ci/dockerfiles/ci-dep
+      #@ build_args = {}
+      #@ build_args["IMAGE_TAG"] = ci_image_tag()
+      #@ build_args["BUILDEVENT_APIKEY"] = '${{ secrets.HONEYCOMB_API_KEY }}'
+      #@ build_args["project"] = dep.name
+      build_args: #@ to_build_args(build_args)
+      tags: (@= ci_image_tag() @)
+#@ end
+
+#@ def ct_suites_job(dep, group):
+name: #@ dep.name + "-ct-" + group["name"]
+needs:
+  - prepare
+  - #@ dep.name + "-checks"
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+  - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+    with:
+      service_account_key: ${{ secrets.GCR_JSON_KEY }}
+      export_default_credentials: true
+  - name: PULL IMAGE
+    run: |
+      gcloud auth configure-docker
+      docker pull (@= ci_dep_image(dep.name) @)
+  #@ for suite in group["suites"]:
+  - name: #@ "RUN ct-" + suite
+    run: |
+      mkdir ct-(@= suite @)-logs && chmod 777 ct-(@= suite @)-logs
+      docker run \
+        --env project=(@= dep.name @) \
+        --env CT_SUITE=(@= suite @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --volume ${PWD}/ct-(@= suite @)-logs:/workspace/ct-logs \
+        (@= ci_dep_image(dep.name) @) \
+        /workspace/rabbitmq/ci/scripts/ct-suite.sh
+  - name: ON FAILURE UPLOAD TESTS LOGS ARTIFACT
+    #! https://github.com/marketplace/actions/upload-artifact
+    uses: actions/upload-artifact@v2-preview
+    #! For whatever reason, "if: steps.(@= suite @).outcome == 'failure'" never executes,
+    #! so just run always
+    if: always()
+    with:
+      name: (@= dep.name @)-ct-(@= suite @)-logs
+      path: "ct-(@= suite @)-logs/*-ct-logs-*.tar.xz"
+  #@ if data.values.erlang_version == data.values.erlang_versions[0]:
+  #@ for version in data.values.secondary_umbrella_gitrefs:
+  #@   logs_dir = 'ct-{}-logs-{}'.format(suite, version)
+  - name: #@ "RUN ct-{} [mixed {}]".format(suite, version)
+    run: |
+      mkdir (@= logs_dir @) && chmod 777 (@= logs_dir @)
+      docker run \
+        --env project=(@= dep.name @) \
+        --env CT_SUITE=(@= suite @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --env SECONDARY_UMBRELLA_VERSION=(@= version @) \
+        --volume ${PWD}/(@= logs_dir @):/workspace/ct-logs \
+        (@= ci_dep_image(dep.name) @) \
+        /workspace/rabbitmq/ci/scripts/ct-suite.sh
+  - name: #@ 'ON FAILURE UPLOAD TESTS LOGS ARTIFACT [mixed {}]'.format(version)
+    #! https://github.com/marketplace/actions/upload-artifact
+    uses: actions/upload-artifact@v2-preview
+    #! For whatever reason, "if: steps.(@= suite @).outcome == 'failure'" never executes,
+    #! so just run always
+    if: always()
+    with:
+      name: (@= dep.name @)-ct-(@= suite @)-logs-mixed-(@= version @)
+      path: "(@= logs_dir @)/*-ct-logs-*.tar.xz"
+  #@ end
+  #@ end
+  #@ end
+#@ end
+
+#@ def collect_job(dep):
+name: #@ dep.name
+needs: #@ [dep.name + "-checks"] + [dep.name + "-ct-" + group["name"] for group in group_by_one(dep.suites)]
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+  - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+    with:
+      service_account_key: ${{ secrets.GCR_JSON_KEY }}
+      export_default_credentials: true
+  - name: RECORD STEP FINISH
+    run: |
+      gcloud auth configure-docker
+      docker pull (@= ci_dep_image(dep.name) @)
+      docker run \
+        --env project=(@= dep.name @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --env STEP_START=${{ needs.(@= dep.name @)-checks.outputs.step_start }} \
+        (@= ci_dep_image(dep.name) @) \
+        /workspace/rabbitmq/ci/scripts/collect.sh
+#@ end
\ No newline at end of file
diff --git a/workflow_sources/test/dep.star b/workflow_sources/test/dep.star
new file mode 100644
index 0000000000..d56e2fe2f6
--- /dev/null
+++ b/workflow_sources/test/dep.star
@@ -0,0 +1,22 @@
+load("util.star", "group_by_one")
+load("rabbitmq_cli.lib.yml", "rabbitmq_cli_job")
+load("ct.lib.yml", "checks_job", "ct_suites_job", "collect_job")
+load("tests.lib.yml", "tests_job")
+
+def dep_jobs(dep):
+  jobs = {}
+  if not getattr(dep, "skip_tests", False):
+    if dep.name == "rabbitmq_cli":
+      jobs[dep.name] = rabbitmq_cli_job(dep)
+    elif getattr(dep, "test_suites_in_parallel", False):
+      jobs[dep.name + "-checks"] = checks_job(dep)
+      for group in group_by_one(dep.suites):
+        jobs[dep.name + "-ct-" + group["name"]] = ct_suites_job(dep, group)
+      end
+      jobs[dep.name] = collect_job(dep)
+    else:
+      jobs[dep.name] = tests_job(dep)
+    end
+  end
+  return jobs
+end
diff --git a/workflow_sources/test/finish.lib.yml b/workflow_sources/test/finish.lib.yml
new file mode 100644
index 0000000000..3d249a7f12
--- /dev/null
+++ b/workflow_sources/test/finish.lib.yml
@@ -0,0 +1,83 @@
+#@ load("@ytt:data", "data")
+#@ load("helpers.star", "ci_image", "skip_ci_condition")
+
+#@ def gcs_path():
+#@   c = ['monorepo_github_actions_conclusions']
+#@   c.append('${{ github.sha }}')
+#@   c.append('${{ github.workflow }}')
+#@   return '/'.join(c)
+#@ end
+
+#@ def finish_jobs(prepare_jobs_names):
+package-generic-unix:
+  name: package-generic-unix
+  needs: [prepare]
+  runs-on: ubuntu-18.04
+  if: #@ skip_ci_condition()
+  #@yaml/text-templated-strings
+  steps:
+    - name: RECORD STEP START
+      id: buildevents
+      run: |
+        echo "::set-output name=step_start::$(date +%s)"
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        service_account_key: ${{ secrets.GCR_JSON_KEY }}
+        export_default_credentials: true
+    - name: PACKAGE GENERIC UNIX
+      run: |
+        gcloud auth configure-docker
+        docker pull (@= ci_image() @)
+        mkdir PACKAGES && chmod 777 PACKAGES
+        docker run \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+          --volume ${PWD}/PACKAGES:/workspace/rabbitmq/PACKAGES \
+          (@= ci_image() @) \
+          ci/scripts/package_generic_unix.sh
+    - name: UPLOAD GENERIC UNIX PACKAGE ARTIFACT
+      #! https://github.com/marketplace/actions/upload-artifact
+      uses: actions/upload-artifact@v2-preview
+      with:
+        name: rabbitmq-server-generic-unix.tar.xz
+        path: "PACKAGES/rabbitmq-server-generic-unix-*.tar.xz"
+    - name: RECORD STEP FINISH
+      if: always()
+      run: |
+        docker run \
+          --env project=packaging \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+          (@= ci_image() @) \
+          ci/scripts/collect.sh
+finish:
+  name: finish
+  needs: #@ prepare_jobs_names + [dep.name for dep in data.values.deps if not getattr(dep, "skip_tests", False)] + ['package-generic-unix']
+  runs-on: ubuntu-18.04
+  if: #@ skip_ci_condition()
+  #@yaml/text-templated-strings
+  steps:
+    - uses: technote-space/workflow-conclusion-action@v1
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        service_account_key: ${{ secrets.GCR_JSON_KEY }}
+        export_default_credentials: true
+    - name: RECORD BUILD FINISH
+      run: |
+        echo -n "${{ env.WORKFLOW_CONCLUSION }}" > conclusion
+
+        gsutil cp conclusion \
+          'gs://(@= gcs_path() @)'
+
+        gcloud auth configure-docker
+        docker pull (@= ci_image() @)
+        docker run \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env BUILD_START=${{ needs.prepare.outputs.build_start }} \
+          --env BUILD_RESULT=${{ env.WORKFLOW_CONCLUSION }} \
+          (@= ci_image() @) \
+          ci/scripts/finish.sh
+#@ end
\ No newline at end of file
diff --git a/workflow_sources/test/helpers.star b/workflow_sources/test/helpers.star
new file mode 100644
index 0000000000..4db571163b
--- /dev/null
+++ b/workflow_sources/test/helpers.star
@@ -0,0 +1,17 @@
+load("@ytt:data", "data")
+
+def ci_image_tag():
+  return "erlang-" + data.values.erlang_version + "-rabbitmq-${{ github.sha }}"
+end
+
+def ci_image():
+  return "eu.gcr.io/cf-rabbitmq-core/ci:" + ci_image_tag()
+end
+
+def ci_dep_image(dep_name):
+  return "eu.gcr.io/cf-rabbitmq-core/ci-" + dep_name + ":" + ci_image_tag()
+end
+
+def skip_ci_condition():
+  return "!contains(github.event.head_commit.message, '[ci skip]')"
+end
diff --git a/workflow_sources/test/prepare.lib.yml b/workflow_sources/test/prepare.lib.yml
new file mode 100644
index 0000000000..da074a0ec6
--- /dev/null
+++ b/workflow_sources/test/prepare.lib.yml
@@ -0,0 +1,146 @@
+#@ load("@ytt:data", "data")
+#@ load("util.star", "to_build_args")
+#@ load("helpers.star", "ci_image", "ci_image_tag", "skip_ci_condition")
+
+#@ def prepare_jobs():
+prepare:
+  name: prepare
+  runs-on: ubuntu-18.04
+  if: #@ skip_ci_condition()
+  outputs:
+    build_start: ${{ steps.buildevents.outputs.build_start }}
+    branch_or_tag_name: ${{ steps.buildevents.outputs.branch_or_tag_name }}
+  #@yaml/text-templated-strings
+  steps:
+    - name: RECORD BUILD START
+      id: buildevents
+      run: |
+        echo "::set-output name=build_start::$(date +%s)"
+        branch_or_tag_name=${GITHUB_REF#refs/*/}
+        echo "::set-output name=branch_or_tag_name::$branch_or_tag_name"
+    - name: CHECKOUT REPOSITORY
+      uses: actions/checkout@v2
+    - name: PREPARE BUILD IMAGE
+      uses: docker/build-push-action@v1
+      with:
+        username: _json_key
+        password: ${{ secrets.GCR_JSON_KEY }}
+        registry: eu.gcr.io
+        repository: cf-rabbitmq-core/ci
+        dockerfile: ci/dockerfiles/ci
+        #@ build_args = {}
+        #@ build_args["ERLANG_VERSION"] = data.values.erlang_version
+        #@ build_args["GITHUB_RUN_ID"] = '${{ github.run_id }}'
+        #@ build_args["BUILDEVENT_APIKEY"] = '${{ secrets.HONEYCOMB_API_KEY }}'
+        #@ build_args["GITHUB_SHA"] = '${{ github.sha }}'
+        #@ build_args["base_rmq_ref"] = data.values.base_rmq_ref
+        #@ build_args["current_rmq_ref"] = '${{ steps.buildevents.outputs.branch_or_tag_name }}'
+        #@ build_args["RABBITMQ_VERSION"] = data.values.base_rmq_ref.replace('master', '3.9.x').replace('.x', '.0')
+        build_args: #@ to_build_args(build_args)
+        tags: (@= ci_image_tag() @)
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        service_account_key: ${{ secrets.GCR_JSON_KEY }}
+        export_default_credentials: true
+    - name: RECORD STEP FINISH
+      run: |
+        docker run \
+          --env project=prepare \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env STEP_START=${{ steps.buildevents.outputs.build_start }} \
+          (@= ci_image() @) \
+          ci/scripts/collect.sh
+xref:
+  name: xref
+  runs-on: ubuntu-18.04
+  if: #@ skip_ci_condition()
+  #@yaml/text-templated-strings
+  steps:
+    - name: RECORD XREF START
+      id: buildevents
+      run: |
+        echo "::set-output name=step_start::$(date +%s)"
+    - name: CHECKOUT REPOSITORY
+      uses: actions/checkout@v2
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        service_account_key: ${{ secrets.GCR_JSON_KEY }}
+        export_default_credentials: true
+    - name: PULL IMAGE
+      #@ base_image = "eu.gcr.io/cf-rabbitmq-core/ci-base:" + data.values.erlang_version
+      run: |
+        gcloud auth configure-docker
+        docker pull (@= base_image @)
+    #@ for dep in [d for d in data.values.deps if not getattr(d, "skip_xref", False)]:
+    - name: RUN XREF (@= dep.name @)
+      run: |
+        docker run \
+          --env project=(@= dep.name @) \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --volume ${PWD}:/workspace/rabbitmq \
+          --workdir /workspace/rabbitmq \
+          (@= base_image @) \
+          ci/scripts/xref.sh
+    #@ end
+    - name: RECORD STEP FINISH
+      if: always()
+      run: |
+        docker run \
+          --env project=xref \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+          --volume ${PWD}/ci/scripts:/workspace/rabbitmq/ci/scripts \
+          --workdir /workspace/rabbitmq \
+          (@= base_image @) \
+          ci/scripts/collect.sh
+
+#@ if/end data.values.erlang_version == data.values.erlang_versions[-1]:
+dialyze:
+  name: dialyze
+  runs-on: ubuntu-18.04
+  if: #@ skip_ci_condition()
+  #@yaml/text-templated-strings
+  steps:
+    - name: RECORD DIALYZE START
+      id: buildevents
+      run: |
+        echo "::set-output name=step_start::$(date +%s)"
+    - name: CHECKOUT REPOSITORY
+      uses: actions/checkout@v2
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        service_account_key: ${{ secrets.GCR_JSON_KEY }}
+        export_default_credentials: true
+    - name: PULL IMAGE
+      #@ base_image = "eu.gcr.io/cf-rabbitmq-core/ci-base:" + data.values.erlang_version
+      run: |
+        gcloud auth configure-docker
+        docker pull (@= base_image @)
+    #@ for dep in [d for d in data.values.deps if not getattr(d, "skip_dialyzer", False)]:
+    - name: RUN DIALYZE (@= dep.name @)
+      run: |
+        docker run \
+          --env project=(@= dep.name @) \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --volume ${PWD}:/workspace/rabbitmq \
+          --workdir /workspace/rabbitmq \
+          (@= base_image @) \
+          ci/scripts/dialyze.sh
+    #@ end
+    - name: RECORD STEP FINISH
+      if: always()
+      run: |
+        docker run \
+          --env project=dialyze \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+          --volume ${PWD}/ci/scripts:/workspace/rabbitmq/ci/scripts \
+          --workdir /workspace/rabbitmq \
+          (@= base_image @) \
+          ci/scripts/collect.sh
+#@ end
\ No newline at end of file
diff --git a/workflow_sources/test/rabbitmq_cli.lib.yml b/workflow_sources/test/rabbitmq_cli.lib.yml
new file mode 100644
index 0000000000..4281547814
--- /dev/null
+++ b/workflow_sources/test/rabbitmq_cli.lib.yml
@@ -0,0 +1,40 @@
+#@ load("@ytt:data", "data")
+#@ load("helpers.star", "ci_image", "skip_ci_condition")
+
+#@ def rabbitmq_cli_job(dep):
+name: #@ dep.name
+needs:
+  - prepare
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+  - name: RECORD STEP START
+    id: buildevents
+    run: |
+      echo "::set-output name=step_start::$(date +%s)"
+  - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+    with:
+      service_account_key: ${{ secrets.GCR_JSON_KEY }}
+      export_default_credentials: true
+  - name: RUN TESTS
+    run: |
+      gcloud auth configure-docker
+      docker pull (@= ci_image() @)
+      mkdir broker-logs && chmod 777 broker-logs
+      docker run \
+        --env project=(@= dep.name @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+        --volume ${PWD}/broker-logs:/workspace/broker-logs \
+        (@= ci_image() @) \
+        ci/scripts/rabbitmq_cli.sh
+  - name: ON FAILURE UPLOAD TESTS LOGS ARTIFACT
+    #! https://github.com/marketplace/actions/upload-artifact
+    uses: actions/upload-artifact@v2-preview
+    if: failure()
+    with:
+      name: (@= dep.name @)-broker-logs
+      path: "broker-logs/broker-logs.tar.xz"
+#@ end
\ No newline at end of file
diff --git a/workflow_sources/test/tests.lib.yml b/workflow_sources/test/tests.lib.yml
new file mode 100644
index 0000000000..f6c7c05912
--- /dev/null
+++ b/workflow_sources/test/tests.lib.yml
@@ -0,0 +1,41 @@
+#@ load("@ytt:data", "data")
+#@ load("util.star", "to_build_args")
+#@ load("helpers.star", "ci_image", "skip_ci_condition")
+
+#@ def tests_job(dep):
+name: #@ dep.name
+needs: [prepare]
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+  - name: RECORD STEP START
+    id: buildevents
+    run: |
+      echo "::set-output name=step_start::$(date +%s)"
+  - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+    with:
+      service_account_key: ${{ secrets.GCR_JSON_KEY }}
+      export_default_credentials: true
+  - name: RUN TESTS
+    run: |
+      gcloud auth configure-docker
+      docker pull (@= ci_image() @)
+      mkdir ct-logs && chmod 777 ct-logs
+      docker run \
+        --env project=(@= dep.name @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+        --volume ${PWD}/ct-logs:/workspace/ct-logs \
+        --oom-kill-disable \
+        (@= ci_image() @) \
+        ci/scripts/tests.sh
+  - name: ON FAILURE UPLOAD TESTS LOGS ARTIFACT
+    #! https://github.com/marketplace/actions/upload-artifact
+    uses: actions/upload-artifact@v2-preview
+    if: failure()
+    with:
+      name: (@= dep.name @)-ct-logs
+      path: "ct-logs/*-ct-logs-*.tar.xz"
+#@ end
\ No newline at end of file
diff --git a/workflow_sources/test/util.star b/workflow_sources/test/util.star
new file mode 100644
index 0000000000..2c6d348564
--- /dev/null
+++ b/workflow_sources/test/util.star
@@ -0,0 +1,58 @@
+def is_unique(l):
+  return len(l) == len(set(l))
+end
+
+def merge(dicts):
+  r = {}
+  for d in dicts:
+    r.update(**d)
+  end
+  return r
+end
+
+def name(suites):
+  if len(suites) == 1:
+    return suites[0].name
+  else:
+    return suites[0].name + "-plus-" + str(len(suites) - 1) + "-more"
+  end
+end
+
+def sum(ints):
+  s = 0
+  for i in ints:
+    s += i
+  end
+  return s
+end
+
+def partition(target, groups, suites):
+  if len(suites) == 0:
+    return groups
+  end
+  group = []
+  rest = []
+  for suite in sorted(suites, key=lambda suite: suite.time):
+    if sum([suite2.time for suite2 in group]) + suite.time <= target:
+      group.append(suite)
+    else:
+      rest.append(suite)
+    end
+  end
+  return partition(target, groups + [group], rest)
+end
+
+def group_by_time(suites):
+  longest = max([suite.time for suite in suites])
+  groups = partition(longest, [], suites)
+  return [{"name": name(suites), "suites": [suite.name for suite in suites]} for suites in groups]
+end
+
+# Used when we don't actually want multiple ct-suites per job
+def group_by_one(suites):
+  return [{"name": suite.name, "suites": [suite.name]} for suite in suites]
+end
+
+def to_build_args(d):
+  return ",".join(['{0}={1}'.format(k,d[k]) for k in d.keys()])
+end
diff --git a/workflow_sources/test/workflow.yml b/workflow_sources/test/workflow.yml
new file mode 100644
index 0000000000..c3746b1944
--- /dev/null
+++ b/workflow_sources/test/workflow.yml
@@ -0,0 +1,16 @@
+#@ load("@ytt:data", "data")
+#@ load("@ytt:template", "template")
+#@ load("util.star", "merge")
+#@ load("prepare.lib.yml", "prepare_jobs")
+#@ load("dep.star", "dep_jobs")
+#@ load("finish.lib.yml", "finish_jobs")
+
+#@ prepare = prepare_jobs()
+#@ deps = [dep_jobs(dep) for dep in data.values.deps]
+#@ finish = finish_jobs([prepare[k]['name'] for k in prepare])
+
+---
+name: #@ "Test - Erlang " + data.values.erlang_version
+#! https://github.com/k14s/ytt/issues/189
+a_magic_string_that_we_will_sed_to_on: push
+jobs: #@ merge([prepare] + deps + [finish])