Mixed version testing in bazel (#3200)

Unlike with gnu make, mixed version testing with bazel uses a package-generic-unix for the secondary umbrella rather than the source. This brings the benefit of being able to mixed version test releases built with older erlang versions (even though all nodes will run under the single version given to bazel)

This introduces new test labels, adding a `-mixed` suffix for every existing test. They can be skipped if necessary with `--test_tag_filters` (see the github actions workflow for an example)

As part of the change, it is now possible to run an old release of rabbit with rabbitmq_run rule, such as:

`bazel run @rabbitmq-server-generic-unix-3.8.17//:rabbitmq-run run-broker`

16 files changed, 316 insertions, 77 deletions

diff --git a/.github/workflows/test-mixed-versions.yaml b/.github/workflows/test-mixed-versions.yaml
new file mode 100644
index 0000000000..5fd2a010fc
--- /dev/null
+++ b/.github/workflows/test-mixed-versions.yaml
@@ -0,0 +1,82 @@
+name: Test Mixed Version Clusters
+on: push
+jobs:
+  test-mixed-versions:
+    name: Test (Mixed Version Cluster)
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        erlang_version:
+        - "23"
+        - "24"
+    timeout-minutes: 120
+    steps:
+    - name: CHECKOUT REPOSITORY
+      uses: actions/checkout@v2.3.4
+    - name: CONFIGURE BAZEL
+      run: |
+        echo "${{ secrets.BUILDBUDDY_CERT }}" > buildbuddy-cert.pem
+        echo "${{ secrets.BUILDBUDDY_KEY }}" > buildbuddy-key.pem
+        cat << EOF >> user.bazelrc
+          build:buildbuddy --tls_client_certificate=buildbuddy-cert.pem
+          build:buildbuddy --tls_client_key=buildbuddy-key.pem
+
+          build:buildbuddy --build_metadata=ROLE=CI
+          build:buildbuddy --build_metadata=VISIBILITY=PUBLIC
+          build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-${{ matrix.erlang_version }}
+        EOF
+    #! - name: Setup tmate session
+    #!   uses: mxschmitt/action-tmate@v3
+    - name: RUN TESTS
+      run: |
+        bazelisk test //... \
+          --config=rbe-${{ matrix.erlang_version }} \
+          --test_tag_filters=mixed-version-cluster,-exclusive,-aws \
+          --verbose_failures
+  test-exclusive-mixed-versions:
+    name: Test (Exclusive Tests with Mixed Version Cluster)
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        erlang_version:
+        - "23"
+        - "24"
+    timeout-minutes: 60
+    steps:
+    - name: CHECKOUT REPOSITORY
+      uses: actions/checkout@v2.3.4
+    - name: CONFIGURE OTP & ELIXIR
+      uses: erlef/setup-beam@v1.8
+      with:
+        otp-version: ${{ matrix.erlang_version }}
+        elixir-version: 1.11.4
+    - name: CONFIGURE BAZEL
+      run: |
+        ERLANG_HOME="$(dirname $(dirname $(which erl)))"
+        ELIXIR_HOME="$(dirname $(dirname $(which iex)))"
+        echo "${{ secrets.BUILDBUDDY_CERT }}" > buildbuddy-cert.pem
+        echo "${{ secrets.BUILDBUDDY_KEY }}" > buildbuddy-key.pem
+        cat << EOF >> user.bazelrc
+          build:buildbuddy --tls_client_certificate=buildbuddy-cert.pem
+          build:buildbuddy --tls_client_key=buildbuddy-key.pem
+
+          build:buildbuddy --build_metadata=ROLE=CI
+          build:buildbuddy --build_metadata=VISIBILITY=PRIVATE
+          build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-exclusive-${{ matrix.erlang_version }}
+
+          build --@bazel-erlang//:erlang_version=${{ matrix.erlang_version }}
+          build --@bazel-erlang//:erlang_home=${ERLANG_HOME}
+          build --//:elixir_home=${ELIXIR_HOME}
+        EOF
+    #! - name: Setup tmate session
+    #!   uses: mxschmitt/action-tmate@v3
+    - name: RUN EXCLUSIVE TESTS
+      run: |
+        MIXED_EXCLUSIVE_TESTS=$(bazel query 'attr(tags, "mixed-version-cluster", attr(tags, "exclusive", tests(//...)))')
+        bazelisk test $MIXED_EXCLUSIVE_TESTS \
+          --config=buildbuddy \
+          --test_tag_filters=-aws \
+          --build_tests_only \
+          --test_env RABBITMQ_CT_HELPERS_DELETE_UNUSED_NODES=true \
+          --verbose_failures
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 333752e239..93584916ae 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -32,7 +32,7 @@ jobs:
       run: |
         bazelisk test //... \
           --config=rbe-${{ matrix.erlang_version }} \
-          --test_tag_filters=-exclusive,-aws \
+          --test_tag_filters=-exclusive,-aws,-mixed-version-cluster \
           --verbose_failures
   test-exclusive:
     name: Test (Exclusive Tests)
@@ -75,7 +75,7 @@ jobs:
       run: |
         bazelisk test //... \
           --config=buildbuddy \
-          --test_tag_filters=exclusive,-aws \
+          --test_tag_filters=exclusive,-aws,-mixed-version-cluster \
           --build_tests_only \
           --test_env RABBITMQ_CT_HELPERS_DELETE_UNUSED_NODES=true \
           --verbose_failures
diff --git a/BUILD.package_generic_unix b/BUILD.package_generic_unix
new file mode 100644
index 0000000000..30286bb112
--- /dev/null
+++ b/BUILD.package_generic_unix
@@ -0,0 +1,18 @@
+load("@//:rabbitmq_package_generic_unix.bzl", "rabbitmq_package_generic_unix")
+load("@//:rabbitmq_run.bzl", "rabbitmq_run")
+
+rabbitmq_package_generic_unix(
+    name = "broker-home",
+    sbin = glob(["sbin/*"]),
+    escript = glob(["escript/*"]),
+    plugins = [
+        "//plugins:standard_plugins",
+        "//plugins:inet_tcp_proxy_ez",
+    ],
+)
+
+rabbitmq_run(
+    name = "rabbitmq-run",
+    home = ":broker-home",
+    visibility = ["//visibility:public"],
+)
diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel
index 466e0f5454..3aa381f754 100644
--- a/WORKSPACE.bazel
+++ b/WORKSPACE.bazel
@@ -17,6 +17,19 @@ load("@io_buildbuddy_buildbuddy_toolchain//:rules.bzl", "buildbuddy")
 buildbuddy(name = "buildbuddy_toolchain")
 
 http_archive(
+    name = "rules_pkg",
+    sha256 = "038f1caa773a7e35b3663865ffb003169c6a71dc995e39bf4815792f385d837d",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
+        "https://github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
+    ],
+)
+
+load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
+rules_pkg_dependencies()
+
+http_archive(
     name = "bazel-erlang",
     strip_prefix = "bazel-erlang-main",
     urls = ["https://github.com/rabbitmq/bazel-erlang/archive/main.zip"],
@@ -51,3 +64,34 @@ git_repository(
 load("//deps/amqp10_client:activemq.bzl", "activemq_archive")
 
 activemq_archive()
+
+ADD_PLUGINS_DIR_BUILD_FILE = """set -euo pipefail
+
+cat << EOF > plugins/BUILD.bazel
+load("@rules_pkg//:pkg.bzl", "pkg_zip")
+
+pkg_zip(
+    name = "inet_tcp_proxy_ez",
+    package_dir = "inet_tcp_proxy/ebin",
+    srcs = [
+        "@inet_tcp_proxy//:bazel_erlang_lib",
+    ],
+    package_file_name = "inet_tcp_proxy.ez",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "standard_plugins",
+    srcs = glob(["*.ez"]),
+    visibility = ["//visibility:public"],
+)
+EOF
+"""
+
+http_archive(
+    name = "rabbitmq-server-generic-unix-3.8.18",
+    build_file = "@//:BUILD.package_generic_unix",
+    patch_cmds = [ADD_PLUGINS_DIR_BUILD_FILE],
+    strip_prefix = "rabbitmq_server-3.8.18",
+    urls = ["https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.8.18/rabbitmq-server-generic-unix-3.8.18.tar.xz"],
+)
diff --git a/deps/rabbit/test/maintenance_mode_SUITE.erl b/deps/rabbit/test/maintenance_mode_SUITE.erl
index c4013e9b32..8b43071bd6 100644
--- a/deps/rabbit/test/maintenance_mode_SUITE.erl
+++ b/deps/rabbit/test/maintenance_mode_SUITE.erl
@@ -10,6 +10,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("amqp_client/include/amqp_client.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl").
 
 -compile(export_all).
 
@@ -247,15 +248,14 @@ quorum_queue_leadership_transfer(Config) ->
                    Config, Nodenames),
     case AllTheSame of
         true ->
-            rabbit_ct_helpers:await_condition(
-              fun () ->
-                      LocalLeaders = rabbit_ct_broker_helpers:rpc(
-                                       Config, A,
-                                       rabbit_amqqueue,
-                                       list_local_leaders,
-                                       []),
-                      length(LocalLeaders) =:= 0
-              end, 20000);
+            ?awaitMatch(
+               LocalLeaders when length(LocalLeaders) == 0,
+                                 rabbit_ct_broker_helpers:rpc(
+                                   Config, A,
+                                   rabbit_amqqueue,
+                                   list_local_leaders,
+                                   []),
+                                 20000);
         false ->
             ct:pal(
               ?LOW_IMPORTANCE,
diff --git a/deps/rabbit/test/rabbit_stream_queue_SUITE.erl b/deps/rabbit/test/rabbit_stream_queue_SUITE.erl
index 8f1fe12833..94811ada4e 100644
--- a/deps/rabbit/test/rabbit_stream_queue_SUITE.erl
+++ b/deps/rabbit/test/rabbit_stream_queue_SUITE.erl
@@ -118,7 +118,17 @@ init_per_suite(Config0) ->
 end_per_suite(Config) ->
     rabbit_ct_helpers:run_teardown_steps(Config).
 
+init_per_group(cluster_size_3_parallel = Group, Config) ->
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "not mixed versions compatible"};
+        _ ->
+            init_per_group1(Group, Config)
+    end;
 init_per_group(Group, Config) ->
+    init_per_group1(Group, Config).
+
+init_per_group1(Group, Config) ->
     ClusterSize = case Group of
                       single_node -> 1;
                       single_node_parallel -> 1;
diff --git a/deps/rabbitmq_federation/test/queue_SUITE.erl b/deps/rabbitmq_federation/test/queue_SUITE.erl
index 3cc3703ce3..cb36827dbf 100644
--- a/deps/rabbitmq_federation/test/queue_SUITE.erl
+++ b/deps/rabbitmq_federation/test/queue_SUITE.erl
@@ -117,10 +117,15 @@ init_per_group(cluster_size_1 = Group, Config) ->
       ]),
     init_per_group1(Group, Config1);
 init_per_group(cluster_size_2 = Group, Config) ->
-    Config1 = rabbit_ct_helpers:set_config(Config, [
-        {rmq_nodes_count, 2}
-      ]),
-    init_per_group1(Group, Config1).
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "not mixed versions compatible"};
+        _ ->
+            Config1 = rabbit_ct_helpers:set_config(Config, [
+                {rmq_nodes_count, 2}
+            ]),
+            init_per_group1(Group, Config1)
+    end.
 
 init_per_group1(Group, Config) ->
     SetupFederation = case Group of
diff --git a/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl b/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl
index 4c5a29e73c..1c003f7ed3 100644
--- a/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl
+++ b/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl
@@ -91,6 +91,10 @@ end_per_group(_, Config) ->
     Steps = Teardown0 ++ Teardown1,
     rabbit_ct_helpers:run_teardown_steps(Config, Steps).
 
+init_per_testcase(Testcase, Config)
+        when Testcase == is_quorum_critical_test
+            orelse Testcase == is_mirror_sync_critical_test ->
+    {skip, "not mixed versions compatible"};
 init_per_testcase(Testcase, Config) ->
     rabbit_ct_helpers:testcase_started(Config, Testcase).
 
diff --git a/deps/rabbitmq_management_agent/test/rabbit_mgmt_gc_SUITE.erl b/deps/rabbitmq_management_agent/test/rabbit_mgmt_gc_SUITE.erl
index 85756da342..b5dc5d39bc 100644
--- a/deps/rabbitmq_management_agent/test/rabbit_mgmt_gc_SUITE.erl
+++ b/deps/rabbitmq_management_agent/test/rabbit_mgmt_gc_SUITE.erl
@@ -78,13 +78,20 @@ end_per_group(_, Config) ->
     Config.
 
 init_per_testcase(quorum_queue_stats = Testcase, Config) ->
-    case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of
-        ok ->
-            rabbit_ct_helpers:testcase_started(Config, Testcase),
-            rabbit_ct_helpers:run_steps(
-              Config, rabbit_ct_client_helpers:setup_steps());
-        Skip ->
-            Skip
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "not mixed versions compatible"};
+        _ ->
+            case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of
+                ok ->
+                    rabbit_ct_helpers:testcase_started(Config, Testcase),
+                    rabbit_ct_helpers:run_steps(
+                    Config, rabbit_ct_client_helpers:setup_steps());
+                {skip, _} = Skip ->
+                    Skip;
+                Other ->
+                    {skip, Other}
+            end
     end;
 init_per_testcase(Testcase, Config) ->
     rabbit_ct_helpers:testcase_started(Config, Testcase),
diff --git a/deps/rabbitmq_peer_discovery_aws/test/integration_SUITE.erl b/deps/rabbitmq_peer_discovery_aws/test/integration_SUITE.erl
index 8b969908e8..6619e04f7e 100644
--- a/deps/rabbitmq_peer_discovery_aws/test/integration_SUITE.erl
+++ b/deps/rabbitmq_peer_discovery_aws/test/integration_SUITE.erl
@@ -42,30 +42,37 @@ groups() ->
     ].
 
 init_per_suite(Config) ->
-    inets:start(),
-    rabbit_ct_helpers:log_environment(),
-    Config1 = rabbit_ct_helpers:set_config(
-                Config, [
-                         {ecs_region, "eu-west-1"},
-                         {ecs_cluster_name, os:getenv("AWS_ECS_CLUSTER_NAME", "rabbitmq-peer-discovery-aws")},
-                         {ecs_profile_name, "rabbitmq-peer-discovery-aws-profile"},
-                         {ecs_instance_role, "ecs-peer-discovery-aws"},
-                         {ecs_cluster_size, ?CLUSTER_SIZE},
-                         {rabbitmq_default_user, "test"},
-                         {rabbitmq_default_pass, rabbit_ct_helpers:random_term_checksum()},
-                         {rabbitmq_erlang_cookie, rabbit_ct_helpers:random_term_checksum()}
-                        ]),
-    Config2 = rabbit_ct_helpers:register_teardown_step(Config1, fun aws_ecs_util:destroy_ecs_cluster/1),
-    rabbit_ct_helpers:run_steps(
-      Config2, [
-                fun rabbit_ct_helpers:init_skip_as_error_flag/1,
-                fun rabbit_ct_helpers:start_long_running_testsuite_monitor/1,
-                fun aws_ecs_util:ensure_aws_cli/1,
-                fun aws_ecs_util:ensure_ecs_cli/1,
-                fun aws_ecs_util:init_aws_credentials/1,
-                fun aws_ecs_util:ensure_rabbitmq_image/1,
-                fun aws_ecs_util:start_ecs_cluster/1
-               ]).
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            %% These test would like passed in mixed versions, but they won't
+            %% actually honor mixed versions as currently specified via env var
+            {skip, "not mixed versions compatible"};
+        _ ->
+            inets:start(),
+            rabbit_ct_helpers:log_environment(),
+            Config1 = rabbit_ct_helpers:set_config(
+                        Config, [
+                                {ecs_region, "eu-west-1"},
+                                {ecs_cluster_name, os:getenv("AWS_ECS_CLUSTER_NAME", "rabbitmq-peer-discovery-aws")},
+                                {ecs_profile_name, "rabbitmq-peer-discovery-aws-profile"},
+                                {ecs_instance_role, "ecs-peer-discovery-aws"},
+                                {ecs_cluster_size, ?CLUSTER_SIZE},
+                                {rabbitmq_default_user, "test"},
+                                {rabbitmq_default_pass, rabbit_ct_helpers:random_term_checksum()},
+                                {rabbitmq_erlang_cookie, rabbit_ct_helpers:random_term_checksum()}
+                                ]),
+            Config2 = rabbit_ct_helpers:register_teardown_step(Config1, fun aws_ecs_util:destroy_ecs_cluster/1),
+            rabbit_ct_helpers:run_steps(
+            Config2, [
+                        fun rabbit_ct_helpers:init_skip_as_error_flag/1,
+                        fun rabbit_ct_helpers:start_long_running_testsuite_monitor/1,
+                        fun aws_ecs_util:ensure_aws_cli/1,
+                        fun aws_ecs_util:ensure_ecs_cli/1,
+                        fun aws_ecs_util:init_aws_credentials/1,
+                        fun aws_ecs_util:ensure_rabbitmq_image/1,
+                        fun aws_ecs_util:start_ecs_cluster/1
+                    ])
+    end.
 
 end_per_suite(Config) ->
     rabbit_ct_helpers:run_teardown_steps(Config).
diff --git a/rabbitmq.bzl b/rabbitmq.bzl
index 944064f3de..80f588dde2 100644
--- a/rabbitmq.bzl
+++ b/rabbitmq.bzl
@@ -5,7 +5,7 @@ load(
     "erlang_lib",
     "test_erlang_lib",
 )
-load("@bazel-erlang//:ct_sharded.bzl", "ct_suite")
+load("@bazel-erlang//:ct_sharded.bzl", "ct_suite", "ct_suite_variant")
 load("//:rabbitmq_home.bzl", "rabbitmq_home")
 load("//:rabbitmq_run.bzl", "rabbitmq_run")
 
@@ -108,15 +108,24 @@ def broker_for_integration_suites():
 
 def rabbitmq_integration_suite(
         package,
+        name = None,
+        tags = [],
         data = [],
         erlc_opts = [],
+        additional_hdrs = [],
+        additional_srcs = [],
         test_env = {},
         tools = [],
         deps = [],
         runtime_deps = [],
         **kwargs):
     ct_suite(
+        name = name,
+        suite_name = name,
+        tags = tags,
         erlc_opts = RABBITMQ_TEST_ERLC_OPTS + erlc_opts,
+        additional_hdrs = additional_hdrs,
+        additional_srcs = additional_srcs,
         data = [
             "@rabbitmq_ct_helpers//tools/tls-certs:Makefile",
             "@rabbitmq_ct_helpers//tools/tls-certs:openssl.cnf.in",
@@ -144,7 +153,42 @@ def rabbitmq_integration_suite(
         ] + deps,
         **kwargs
     )
-    return kwargs["name"]
+
+    ct_suite_variant(
+        name = name + "-mixed",
+        suite_name = name,
+        tags = tags + ["mixed-version-cluster"],
+        data = [
+            "@rabbitmq_ct_helpers//tools/tls-certs:Makefile",
+            "@rabbitmq_ct_helpers//tools/tls-certs:openssl.cnf.in",
+        ] + data,
+        test_env = dict({
+            "SKIP_MAKE_TEST_DIST": "true",
+            "RABBITMQ_FEATURE_FLAGS": "",
+            "RABBITMQ_RUN": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/rabbitmq-for-tests-run".format(package),
+            "RABBITMQCTL": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/broker-for-tests-home/sbin/rabbitmqctl".format(package),
+            "RABBITMQ_PLUGINS": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/broker-for-tests-home/sbin/rabbitmq-plugins".format(package),
+            "RABBITMQ_QUEUES": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/broker-for-tests-home/sbin/rabbitmq-queues".format(package),
+            "RABBITMQ_RUN_SECONDARY": "$TEST_SRCDIR/rabbitmq-server-generic-unix-3.8.18/rabbitmq-run",
+        }.items() + test_env.items()),
+        tools = [
+            ":rabbitmq-for-tests-run",
+            "@rabbitmq-server-generic-unix-3.8.18//:rabbitmq-run",
+        ] + tools,
+        runtime_deps = [
+            "//deps/rabbitmq_cli:elixir_as_bazel_erlang_lib",
+            "//deps/rabbitmq_cli:rabbitmqctl",
+            "@rabbitmq_ct_client_helpers//:bazel_erlang_lib",
+        ] + runtime_deps,
+        deps = [
+            "//deps/amqp_client:bazel_erlang_lib",
+            "//deps/rabbit_common:bazel_erlang_lib",
+            "@rabbitmq_ct_helpers//:bazel_erlang_lib",
+        ] + deps,
+        **kwargs
+    )
+
+    return name
 
 def assert_suites(suite_names, suite_files):
     for f in suite_files:
diff --git a/rabbitmq_home.bzl b/rabbitmq_home.bzl
index 060101c425..e23ae40f20 100644
--- a/rabbitmq_home.bzl
+++ b/rabbitmq_home.bzl
@@ -6,7 +6,6 @@ RabbitmqHomeInfo = provider(
         "sbin": "Files making up the sbin dir",
         "escript": "Files making up the escript dir",
         "plugins": "Files making up the plugins dir",
-        "erlang_version": "Version of the Erlang compiler used",
     },
 )
 
@@ -118,7 +117,6 @@ def _impl(ctx):
             sbin = scripts,
             escript = escripts,
             plugins = plugins,
-            erlang_version = erlang_versions[0],
         ),
         DefaultInfo(
             files = depset(scripts + escripts + plugins),
@@ -141,7 +139,6 @@ rabbitmq_home = rule(
             allow_files = True,
         ),
         "_rabbitmqctl_escript": attr.label(default = "//deps/rabbitmq_cli:rabbitmqctl"),
-        "_erlang_version": attr.label(default = "@bazel-erlang//:erlang_version"),
         "plugins": attr.label_list(),
     },
 )
diff --git a/rabbitmq_package_generic_unix.bzl b/rabbitmq_package_generic_unix.bzl
new file mode 100644
index 0000000000..392c18c8ba
--- /dev/null
+++ b/rabbitmq_package_generic_unix.bzl
@@ -0,0 +1,26 @@
+load("@//:rabbitmq_home.bzl", "RabbitmqHomeInfo")
+
+def _impl(ctx):
+    scripts = ctx.files.sbin
+    escripts = ctx.files.escript
+    plugins = ctx.files.plugins
+
+    return [
+        RabbitmqHomeInfo(
+            sbin = scripts,
+            escript = escripts,
+            plugins = plugins,
+        ),
+        DefaultInfo(
+            files = depset(scripts + escripts + plugins),
+        ),
+    ]
+
+rabbitmq_package_generic_unix = rule(
+    implementation = _impl,
+    attrs = {
+        "sbin": attr.label_list(allow_files = True),
+        "escript": attr.label_list(allow_files = True),
+        "plugins": attr.label_list(allow_files = True),
+    },
+)
diff --git a/rabbitmq_run.bzl b/rabbitmq_run.bzl
index 4bb7c77c56..f01fef395c 100644
--- a/rabbitmq_run.bzl
+++ b/rabbitmq_run.bzl
@@ -3,38 +3,34 @@ load("@bazel-erlang//:bazel_erlang_lib.bzl", "path_join")
 load("@bazel-erlang//:ct.bzl", "sanitize_sname")
 load(":rabbitmq_home.bzl", "RabbitmqHomeInfo")
 
-# Note: Theses rules take advantage of the fact that when the files from
-#       the rabbitmq_home rule are used as runfiles, they are linked in
-#       at their declared relative paths. In other words, since
-#       rabbitmq_home declares "sbin/rabbitmq-server", is still at
-#       "sbin/rabbitmq-server" when our script runs.
+def _dirname(p):
+    return p.rpartition("/")[0]
+
+def _rabbitmq_home_info_root_short_path(rabbitmq_home):
+    return _dirname(_dirname(rabbitmq_home.sbin[0].short_path))
 
 def _impl(ctx):
-    erlang_version = ctx.attr._erlang_version[ErlangVersionProvider].version
     rabbitmq_home = ctx.attr.home[RabbitmqHomeInfo]
 
-    if rabbitmq_home.erlang_version != erlang_version:
-        fail("Mismatched erlang versions", erlang_version, rabbitmq_home.erlang_version)
+    root = _rabbitmq_home_info_root_short_path(rabbitmq_home)
 
-    erl_libs = ":".join(
-        [p.short_path for p in rabbitmq_home.plugins],
-    )
+    erl_libs = [path_join(root, "plugins")]
 
     ctx.actions.expand_template(
         template = ctx.file._template,
         output = ctx.outputs.executable,
         substitutions = {
-            "{RABBITMQ_HOME}": ctx.attr.home.label.name,
-            "{ERL_LIBS}": erl_libs,
+            "{RABBITMQ_HOME}": root,
+            "{ERL_LIBS}": ":".join(erl_libs),
             "{ERLANG_HOME}": ctx.attr._erlang_home[ErlangHomeProvider].path,
             "{SNAME}": sanitize_sname("sbb-" + ctx.attr.name),
         },
         is_executable = True,
     )
 
-    return [DefaultInfo(
-        runfiles = ctx.runfiles(ctx.attr.home[DefaultInfo].files.to_list()),
-    )]
+    runfiles = ctx.runfiles(ctx.attr.home[DefaultInfo].files.to_list())
+
+    return [DefaultInfo(runfiles = runfiles)]
 
 rabbitmq_run = rule(
     implementation = _impl,
@@ -44,7 +40,6 @@ rabbitmq_run = rule(
             allow_single_file = True,
         ),
         "_erlang_home": attr.label(default = "@bazel-erlang//:erlang_home"),
-        "_erlang_version": attr.label(default = "@bazel-erlang//:erlang_version"),
         "home": attr.label(providers = [RabbitmqHomeInfo]),
     },
     executable = True,
diff --git a/rabbitmqctl.bzl b/rabbitmqctl.bzl
index ec31d115ff..be8c90462a 100644
--- a/rabbitmqctl.bzl
+++ b/rabbitmqctl.bzl
@@ -6,9 +6,6 @@ def _impl(ctx):
 
     rabbitmq_home = ctx.attr.home[RabbitmqHomeInfo]
 
-    if rabbitmq_home.erlang_version != erlang_version:
-        fail("Mismatched erlang versions", erlang_version, rabbitmq_home.erlang_version)
-
     script = """
     exec ./{home}/sbin/{cmd} $@
     """.format(
diff --git a/scripts/bazel/rabbitmq-run.sh b/scripts/bazel/rabbitmq-run.sh
index 3992a78602..2369f024f9 100644
--- a/scripts/bazel/rabbitmq-run.sh
+++ b/scripts/bazel/rabbitmq-run.sh
@@ -1,8 +1,11 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# https://stackoverflow.com/a/4774063
-SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+if [ -z ${TEST_SRCDIR+x} ]; then
+BASE_DIR=$PWD
+else
+BASE_DIR=$TEST_SRCDIR/$TEST_WORKSPACE
+fi
 
 if [ $1 = "-C" ]; then
     cd $2
@@ -32,13 +35,13 @@ for arg in "$@"; do
     esac
 done
 
-DEFAULT_PLUGINS_DIR=${SCRIPTPATH}/{RABBITMQ_HOME}/plugins
+DEFAULT_PLUGINS_DIR=${BASE_DIR}/{RABBITMQ_HOME}/plugins
 if [ ! -z ${EXTRA_PLUGINS_DIR+x} ]; then
     DEFAULT_PLUGINS_DIR=${DEFAULT_PLUGINS_DIR}:${EXTRA_PLUGINS_DIR}
 fi
 
 TEST_TMPDIR=${TEST_TMPDIR:=${TMPDIR}/rabbitmq-test-instances}
-RABBITMQ_SCRIPTS_DIR=${SCRIPTPATH}/{RABBITMQ_HOME}/sbin
+RABBITMQ_SCRIPTS_DIR=${BASE_DIR}/{RABBITMQ_HOME}/sbin
 RABBITMQ_PLUGINS=${RABBITMQ_SCRIPTS_DIR}/rabbitmq-plugins
 RABBITMQ_SERVER=${RABBITMQ_SCRIPTS_DIR}/rabbitmq-server
 RABBITMQCTL=${RABBITMQ_SCRIPTS_DIR}/rabbitmqzctl
@@ -161,12 +164,12 @@ case $CMD in
             while ps -p "$pid" >/dev/null 2>&1; do sleep 1; done
         ;;
     set-resource-alarm)
-        ERL_LIBS="{ERL_LIBS}" \
+        ERL_LIBS="${BASE_DIR}/{ERL_LIBS}" \
            ${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl -n ${RABBITMQ_NODENAME} \
             eval "rabbit_alarm:set_alarm({{resource_limit, ${SOURCE}, node()}, []})."
         ;;
     clear-resource-alarm)
-        ERL_LIBS="{ERL_LIBS}" \
+        ERL_LIBS="${BASE_DIR}/{ERL_LIBS}" \
             ${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl -n ${RABBITMQ_NODENAME} \
             eval "rabbit_alarm:clear_alarm({resource_limit, ${SOURCE}, node()})."
         ;;