Revert "Merge pull request #3683 from rabbitmq/use-bazel-toolchains"

This reverts commit 82508ba167e94d42067e7562a0589b1d977d6a86, reversing
changes made to 115b951b9cc718fd40aa560e49319f57c762f05d.

8 files changed, 66 insertions, 106 deletions

diff --git a/.bazelrc b/.bazelrc
index 5b11a54475..af4ae18855 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -14,25 +14,38 @@ build:rbe --config=buildbuddy
 
 build:rbe --remote_executor=grpcs://remote.buildbuddy.io
 
-build:rbe --host_javabase=@rbe_default//java:jdk
-build:rbe --javabase=@rbe_default//java:jdk
-build:rbe --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
-build:rbe --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
-build:rbe --crosstool_top=@rbe_default//cc:toolchain
-build:rbe --extra_toolchains=@rbe_default//config:cc-toolchain
-build:rbe --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
-
-build:rbe --host_platform=@rbe_default//config:platform
-build:rbe --platforms=@rbe_default//config:platform
-build:rbe --extra_execution_platforms=@rbe_default//config:platform
-
-build:rbe --@bazel-erlang//:erlang_version=24
+build:rbe --crosstool_top=@buildbuddy_toolchain//:toolchain
+build:rbe --extra_toolchains=@buildbuddy_toolchain//:cc_toolchain
+build:rbe --javabase=@buildbuddy_toolchain//:javabase_jdk8
+build:rbe --host_javabase=@buildbuddy_toolchain//:javabase_jdk8
+build:rbe --java_toolchain=@buildbuddy_toolchain//:toolchain_jdk8
+build:rbe --host_java_toolchain=@buildbuddy_toolchain//:toolchain_jdk8
+
 build:rbe --@bazel-erlang//:erlang_home=/usr/lib/erlang
 build:rbe --//:elixir_home=/usr/local
 
 build:rbe --spawn_strategy=remote
 build:rbe --test_strategy=""
-build:rbe --jobs=100
+build:rbe --jobs=50
+
+build:rbe-23 --config=rbe
+build:rbe-23 --host_platform=//:erlang_23_platform
+build:rbe-23 --platforms=//:erlang_23_platform
+build:rbe-23 --extra_execution_platforms=//:erlang_23_platform
+build:rbe-23 --@bazel-erlang//:erlang_version=23
+
+build:rbe-24 --config=rbe
+build:rbe-24 --host_platform=//:erlang_24_platform
+build:rbe-24 --platforms=//:erlang_24_platform
+build:rbe-24 --extra_execution_platforms=//:erlang_24_platform
+build:rbe-24 --@bazel-erlang//:erlang_version=24
+
+build:rbe-git --config=rbe
+build:rbe-git --host_platform=//:erlang_git_platform
+build:rbe-git --platforms=//:erlang_git_platform
+build:rbe-git --extra_execution_platforms=//:erlang_git_platform
+build:rbe-git --@bazel-erlang//:erlang_home=/usr/local/lib/erlang
+build:rbe-git --@bazel-erlang//:erlang_version=25
 
 # Try importing a user specific .bazelrc
 # You can create your own by copying and editing the template-user.bazelrc template:
diff --git a/.github/workflows/perform-bazel-execution-comparison.yaml b/.github/workflows/perform-bazel-execution-comparison.yaml
index 5f13a616fa..4834cd1bf0 100644
--- a/.github/workflows/perform-bazel-execution-comparison.yaml
+++ b/.github/workflows/perform-bazel-execution-comparison.yaml
@@ -6,41 +6,29 @@ on:
         description: 'A bazel label representing the test target'
         required: true
         default: '//deps/rabbit:rabbit_stream_queue_SUITE'
+env:
+  ERLANG_MAJOR: "24"
+  CACHE_NAME: ci-bazel-cache-analysis
 jobs:
   run-a:
     name: Run A
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        erlang_version:
-        - "24"
-        include:
-        - erlang_version: "24"
-          rbe_default_branch: linux-erlang-24.1
-          cache_name: ci-bazel-cache-analysis
     timeout-minutes: 120
     steps:
     - name: CHECKOUT REPOSITORY
       uses: actions/checkout@v2.4.0
-    - name: SELECT ERLANG VERSION
-      run: |
-        sudo npm install --global --silent @bazel/buildozer
-        echo "$(cat WORKSPACE.bazel | npx buildozer 'set branch "${{ matrix.rbe_default_branch }}"' -:rbe_default)" > WORKSPACE.bazel
-        git diff
     - name: CONFIGURE BAZEL
       run: |
         cat << EOF >> user.bazelrc
           build:buildbuddy --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_API_KEY }}
 
           build:buildbuddy --build_metadata=ROLE=CI
-          build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-${{ matrix.cache_name }}
-
-          build:rbe --@bazel-erlang//:erlang_version=${{ matrix.erlang_version }}
+          build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-${CACHE_NAME}
         EOF
     - name: RUN TESTS
       run: |
         bazelisk test ${{ github.event.inputs.target }} \
-          --config=rbe \
+          --config=rbe-${ERLANG_MAJOR} \
           --execution_log_binary_file=/tmp/exec.log
     - name: SAVE EXECUTION LOG BINARY
       uses: actions/upload-artifact@v2-preview
@@ -51,37 +39,22 @@ jobs:
     name: Run B
     needs: run-a
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        erlang_version:
-        - "24"
-        include:
-        - erlang_version: "24"
-          rbe_default_branch: linux-erlang-24.1
-          cache_name: ci-bazel-cache-analysis
     timeout-minutes: 120
     steps:
     - name: CHECKOUT REPOSITORY
       uses: actions/checkout@v2.4.0
-    - name: SELECT ERLANG VERSION
-      run: |
-        sudo npm install --global --silent @bazel/buildozer
-        echo "$(cat WORKSPACE.bazel | npx buildozer 'set branch "${{ matrix.rbe_default_branch }}"' -:rbe_default)" > WORKSPACE.bazel
-        git diff
     - name: CONFIGURE BAZEL
       run: |
         cat << EOF >> user.bazelrc
           build:buildbuddy --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_API_KEY }}
 
           build:buildbuddy --build_metadata=ROLE=CI
-          build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-${{ matrix.cache_name }}
-
-          build:rbe --@bazel-erlang//:erlang_version=${{ matrix.erlang_version }}
+          build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-${CACHE_NAME}
         EOF
     - name: RUN TESTS
       run: |
         bazelisk test ${{ github.event.inputs.target }} \
-          --config=rbe \
+          --config=rbe-${ERLANG_MAJOR} \
           --execution_log_binary_file=/tmp/exec.log
     - name: SAVE EXECUTION LOG BINARY
       uses: actions/upload-artifact@v2-preview
diff --git a/.github/workflows/rabbitmq_peer_discovery_aws.yaml b/.github/workflows/rabbitmq_peer_discovery_aws.yaml
index b8f177accd..afc8a34be2 100644
--- a/.github/workflows/rabbitmq_peer_discovery_aws.yaml
+++ b/.github/workflows/rabbitmq_peer_discovery_aws.yaml
@@ -20,7 +20,6 @@ jobs:
         include:
         - image_tag_suffix: otp-max
           erlang_version: "24"
-          rbe_default_branch: linux-erlang-24.1
     timeout-minutes: 45
     steps:
     - name: CHECKOUT REPOSITORY
@@ -32,11 +31,6 @@ jobs:
         check-name: build-publish-dev (${{ matrix.image_tag_suffix }})
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         wait-interval: 30 # seconds
-    - name: SELECT ERLANG VERSION
-      run: |
-        sudo npm install --global --silent @bazel/buildozer
-        echo "$(cat WORKSPACE.bazel | npx buildozer 'set branch "${{ matrix.rbe_default_branch }}"' -:rbe_default)" > WORKSPACE.bazel
-        git diff
     - name: CONFIGURE BAZEL
       run: |
         cat << EOF >> user.bazelrc
@@ -45,8 +39,6 @@ jobs:
           build:buildbuddy --build_metadata=ROLE=CI
           build:buildbuddy --build_metadata=VISIBILITY=PRIVATE
           build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-aws-${{ matrix.erlang_version }}
-
-          build:rbe --@bazel-erlang//:erlang_version=${{ matrix.erlang_version }}
         EOF
     #! - name: Setup tmate session
     #!   uses: mxschmitt/action-tmate@v3
@@ -54,7 +46,7 @@ jobs:
       run: |
         branch_or_tag="${GITHUB_REF##*/}"
         bazelisk test //deps/rabbitmq_peer_discovery_aws:integration_SUITE \
-          --config=rbe \
+          --config=rbe-${{ matrix.erlang_version }} \
           --test_tag_filters=aws \
           --build_tests_only \
           --test_env AWS_ACCESS_KEY_ID=${{ secrets.CONCOURSE_AWS_ACCESS_KEY_ID }} \
diff --git a/.github/workflows/test-erlang-git.yaml b/.github/workflows/test-erlang-git.yaml
index 2b05348b84..8a6161ac48 100644
--- a/.github/workflows/test-erlang-git.yaml
+++ b/.github/workflows/test-erlang-git.yaml
@@ -32,19 +32,6 @@ jobs:
           build:buildbuddy --build_metadata=ROLE=CI
           build:buildbuddy --build_metadata=VISIBILITY=PUBLIC
           build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-erlang-git
-
-          build:rbe-git --crosstool_top=@buildbuddy_toolchain//:toolchain
-          build:rbe-git --extra_toolchains=@buildbuddy_toolchain//:cc_toolchain
-          build:rbe-git --javabase=@buildbuddy_toolchain//:javabase_jdk8
-          build:rbe-git --host_javabase=@buildbuddy_toolchain//:javabase_jdk8
-          build:rbe-git --java_toolchain=@buildbuddy_toolchain//:toolchain_jdk8
-          build:rbe-git --host_java_toolchain=@buildbuddy_toolchain//:toolchain_jdk8
-
-          build:rbe-git --host_platform=//:erlang_git_platform
-          build:rbe-git --platforms=//:erlang_git_platform
-          build:rbe-git --extra_execution_platforms=//:erlang_git_platform
-          build:rbe-git --@bazel-erlang//:erlang_home=/usr/local/lib/erlang
-          build:rbe-git --@bazel-erlang//:erlang_version=25
         EOF
     #! - name: Setup tmate session
     #!   uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/test-mixed-versions.yaml b/.github/workflows/test-mixed-versions.yaml
index 6351659dc4..abf2584a9f 100644
--- a/.github/workflows/test-mixed-versions.yaml
+++ b/.github/workflows/test-mixed-versions.yaml
@@ -31,20 +31,10 @@ jobs:
         erlang_version:
         - "23"
         - "24"
-        include:
-        - erlang_version: "23"
-          rbe_default_branch: linux-erlang-23.3
-        - erlang_version: "24"
-          rbe_default_branch: linux-erlang-24.1
     timeout-minutes: 120
     steps:
     - name: CHECKOUT REPOSITORY
       uses: actions/checkout@v2.4.0
-    - name: SELECT ERLANG VERSION
-      run: |
-        sudo npm install --global --silent @bazel/buildozer
-        echo "$(cat WORKSPACE.bazel | npx buildozer 'set branch "${{ matrix.rbe_default_branch }}"' -:rbe_default)" > WORKSPACE.bazel
-        git diff
     - name: CONFIGURE BAZEL
       run: |
         cat << EOF >> user.bazelrc
@@ -53,15 +43,13 @@ jobs:
           build:buildbuddy --build_metadata=ROLE=CI
           build:buildbuddy --build_metadata=VISIBILITY=PUBLIC
           build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-${{ matrix.erlang_version }}
-
-          build:rbe --@bazel-erlang//:erlang_version=${{ matrix.erlang_version }}
         EOF
     #! - name: Setup tmate session
     #!   uses: mxschmitt/action-tmate@v3
     - name: RUN TESTS
       run: |
         bazelisk test //... \
-          --config=rbe \
+          --config=rbe-${{ matrix.erlang_version }} \
           --test_tag_filters=mixed-version-cluster,-exclusive,-aws \
           --verbose_failures
   test-exclusive-mixed-versions:
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 2f35eebb07..b7f7ad6165 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -23,20 +23,10 @@ jobs:
         erlang_version:
         - "23"
         - "24"
-        include:
-        - erlang_version: "23"
-          rbe_default_branch: linux-erlang-23.3
-        - erlang_version: "24"
-          rbe_default_branch: linux-erlang-24.1
     timeout-minutes: 120
     steps:
     - name: CHECKOUT REPOSITORY
       uses: actions/checkout@v2.4.0
-    - name: SELECT ERLANG VERSION
-      run: |
-        sudo npm install --global --silent @bazel/buildozer
-        echo "$(cat WORKSPACE.bazel | npx buildozer 'set branch "${{ matrix.rbe_default_branch }}"' -:rbe_default)" > WORKSPACE.bazel
-        git diff
     - name: CONFIGURE BAZEL
       run: |
         cat << EOF >> user.bazelrc
@@ -45,8 +35,6 @@ jobs:
           build:buildbuddy --build_metadata=ROLE=CI
           build:buildbuddy --build_metadata=VISIBILITY=PUBLIC
           build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-${{ matrix.erlang_version }}
-
-          build:rbe --@bazel-erlang//:erlang_version=${{ matrix.erlang_version }}
         EOF
 
         bazelisk info release
@@ -55,7 +43,7 @@ jobs:
     - name: RUN TESTS
       run: |
         bazelisk test //... \
-          --config=rbe \
+          --config=rbe-${{ matrix.erlang_version }} \
           --test_tag_filters=-exclusive,-aws,-mixed-version-cluster \
           --verbose_failures
   test-exclusive:
diff --git a/BUILD.bazel b/BUILD.bazel
index 9ea70fc6e3..95e4d1fa1c 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -19,6 +19,34 @@ elixir_home(
 )
 
 platform(
+    name = "erlang_23_platform",
+    constraint_values = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_64",
+        "@bazel_tools//tools/cpp:clang",
+    ],
+    exec_properties = {
+        "OSFamily": "Linux",
+        # linux-erlang-23.3
+        "container-image": "docker://pivotalrabbitmq/rabbitmq-server-buildenv@sha256:5de95518e8d5f3724839ad46e450b80d89cb0e7e546872a63b7ce4fd482a696e",
+    },
+)
+
+platform(
+    name = "erlang_24_platform",
+    constraint_values = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_64",
+        "@bazel_tools//tools/cpp:clang",
+    ],
+    exec_properties = {
+        "OSFamily": "Linux",
+        # linux-erlang-24.0
+        "container-image": "docker://pivotalrabbitmq/rabbitmq-server-buildenv@sha256:52a81330352656180952e9c3f09e510a529cc0ed4fd6bc3b480ad313f2ddc3ae",
+    },
+)
+
+platform(
     name = "erlang_git_platform",
     constraint_values = [
         "@platforms//os:linux",
diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel
index fa0b615a27..dbb54fd1f9 100644
--- a/WORKSPACE.bazel
+++ b/WORKSPACE.bazel
@@ -14,16 +14,7 @@ buildbuddy_deps()
 
 load("@io_buildbuddy_buildbuddy_toolchain//:rules.bzl", "buildbuddy")
 
-buildbuddy(
-    name = "buildbuddy_toolchain",
-    llvm = True,
-)
-
-git_repository(
-    name = "rbe_default",
-    branch = "linux-erlang-24.1",
-    remote = "https://github.com/rabbitmq/rbe-erlang-platform.git",
-)
+buildbuddy(name = "buildbuddy_toolchain")
 
 http_archive(
     name = "rules_pkg",