Mixed version testing in bazel (#3200)

Unlike with gnu make, mixed version testing with bazel uses a package-generic-unix for the secondary umbrella rather than the source. This brings the benefit of being able to mixed version test releases built with older erlang versions (even though all nodes will run under the single version given to bazel)

This introduces new test labels, adding a `-mixed` suffix for every existing test. They can be skipped if necessary with `--test_tag_filters` (see the github actions workflow for an example)

As part of the change, it is now possible to run an old release of rabbit with rabbitmq_run rule, such as:

`bazel run @rabbitmq-server-generic-unix-3.8.17//:rabbitmq-run run-broker`

(cherry picked from commit d6399bbb5b6c90c2a9e3d20011452ea59808c341)
(cherry picked from commit 6f24943f038b40a0bc368c344cc0b1dd8f4b9f93)

17 files changed, 348 insertions, 93 deletions

diff --git a/.github/workflows/test-mixed-versions.yaml b/.github/workflows/test-mixed-versions.yaml
new file mode 100644
index 0000000000..5fd2a010fc
--- /dev/null
+++ b/.github/workflows/test-mixed-versions.yaml
@@ -0,0 +1,82 @@
+name: Test Mixed Version Clusters
+on: push
+jobs:
+  test-mixed-versions:
+    name: Test (Mixed Version Cluster)
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        erlang_version:
+        - "23"
+        - "24"
+    timeout-minutes: 120
+    steps:
+    - name: CHECKOUT REPOSITORY
+      uses: actions/checkout@v2.3.4
+    - name: CONFIGURE BAZEL
+      run: |
+        echo "${{ secrets.BUILDBUDDY_CERT }}" > buildbuddy-cert.pem
+        echo "${{ secrets.BUILDBUDDY_KEY }}" > buildbuddy-key.pem
+        cat << EOF >> user.bazelrc
+          build:buildbuddy --tls_client_certificate=buildbuddy-cert.pem
+          build:buildbuddy --tls_client_key=buildbuddy-key.pem
+
+          build:buildbuddy --build_metadata=ROLE=CI
+          build:buildbuddy --build_metadata=VISIBILITY=PUBLIC
+          build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-${{ matrix.erlang_version }}
+        EOF
+    #! - name: Setup tmate session
+    #!   uses: mxschmitt/action-tmate@v3
+    - name: RUN TESTS
+      run: |
+        bazelisk test //... \
+          --config=rbe-${{ matrix.erlang_version }} \
+          --test_tag_filters=mixed-version-cluster,-exclusive,-aws \
+          --verbose_failures
+  test-exclusive-mixed-versions:
+    name: Test (Exclusive Tests with Mixed Version Cluster)
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        erlang_version:
+        - "23"
+        - "24"
+    timeout-minutes: 60
+    steps:
+    - name: CHECKOUT REPOSITORY
+      uses: actions/checkout@v2.3.4
+    - name: CONFIGURE OTP & ELIXIR
+      uses: erlef/setup-beam@v1.8
+      with:
+        otp-version: ${{ matrix.erlang_version }}
+        elixir-version: 1.11.4
+    - name: CONFIGURE BAZEL
+      run: |
+        ERLANG_HOME="$(dirname $(dirname $(which erl)))"
+        ELIXIR_HOME="$(dirname $(dirname $(which iex)))"
+        echo "${{ secrets.BUILDBUDDY_CERT }}" > buildbuddy-cert.pem
+        echo "${{ secrets.BUILDBUDDY_KEY }}" > buildbuddy-key.pem
+        cat << EOF >> user.bazelrc
+          build:buildbuddy --tls_client_certificate=buildbuddy-cert.pem
+          build:buildbuddy --tls_client_key=buildbuddy-key.pem
+
+          build:buildbuddy --build_metadata=ROLE=CI
+          build:buildbuddy --build_metadata=VISIBILITY=PRIVATE
+          build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-exclusive-${{ matrix.erlang_version }}
+
+          build --@bazel-erlang//:erlang_version=${{ matrix.erlang_version }}
+          build --@bazel-erlang//:erlang_home=${ERLANG_HOME}
+          build --//:elixir_home=${ELIXIR_HOME}
+        EOF
+    #! - name: Setup tmate session
+    #!   uses: mxschmitt/action-tmate@v3
+    - name: RUN EXCLUSIVE TESTS
+      run: |
+        MIXED_EXCLUSIVE_TESTS=$(bazel query 'attr(tags, "mixed-version-cluster", attr(tags, "exclusive", tests(//...)))')
+        bazelisk test $MIXED_EXCLUSIVE_TESTS \
+          --config=buildbuddy \
+          --test_tag_filters=-aws \
+          --build_tests_only \
+          --test_env RABBITMQ_CT_HELPERS_DELETE_UNUSED_NODES=true \
+          --verbose_failures
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index e3361dddbe..f915cbd930 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -32,7 +32,7 @@ jobs:
       run: |
         bazelisk test //... \
           --config=rbe-${{ matrix.erlang_version }} \
-          --test_tag_filters=-exclusive,-aws \
+          --test_tag_filters=-exclusive,-aws,-mixed-version-cluster \
           --verbose_failures
   test-exclusive:
     name: Test (Exclusive Tests)
@@ -75,7 +75,7 @@ jobs:
       run: |
         bazelisk test //... \
           --config=buildbuddy \
-          --test_tag_filters=exclusive,-aws \
+          --test_tag_filters=exclusive,-aws,-mixed-version-cluster \
           --build_tests_only \
           --test_env RABBITMQ_CT_HELPERS_DELETE_UNUSED_NODES=true \
           --verbose_failures
diff --git a/BUILD.package_generic_unix b/BUILD.package_generic_unix
new file mode 100644
index 0000000000..30286bb112
--- /dev/null
+++ b/BUILD.package_generic_unix
@@ -0,0 +1,18 @@
+load("@//:rabbitmq_package_generic_unix.bzl", "rabbitmq_package_generic_unix")
+load("@//:rabbitmq_run.bzl", "rabbitmq_run")
+
+rabbitmq_package_generic_unix(
+    name = "broker-home",
+    sbin = glob(["sbin/*"]),
+    escript = glob(["escript/*"]),
+    plugins = [
+        "//plugins:standard_plugins",
+        "//plugins:inet_tcp_proxy_ez",
+    ],
+)
+
+rabbitmq_run(
+    name = "rabbitmq-run",
+    home = ":broker-home",
+    visibility = ["//visibility:public"],
+)
diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel
index 2de54bdb86..d990e57a44 100644
--- a/WORKSPACE.bazel
+++ b/WORKSPACE.bazel
@@ -17,6 +17,19 @@ load("@io_buildbuddy_buildbuddy_toolchain//:rules.bzl", "buildbuddy")
 buildbuddy(name = "buildbuddy_toolchain")
 
 http_archive(
+    name = "rules_pkg",
+    sha256 = "038f1caa773a7e35b3663865ffb003169c6a71dc995e39bf4815792f385d837d",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
+        "https://github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
+    ],
+)
+
+load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
+rules_pkg_dependencies()
+
+http_archive(
     name = "bazel-erlang",
     sha256 = "422a9222522216f59a01703a13f578c601d6bddf5617bee8da3c43e3b299fc4e",
     strip_prefix = "bazel-erlang-1.1.0",
@@ -52,3 +65,35 @@ git_repository(
 load("//deps/amqp10_client:activemq.bzl", "activemq_archive")
 
 activemq_archive()
+
+ADD_PLUGINS_DIR_BUILD_FILE = """set -euo pipefail
+
+cat << EOF > plugins/BUILD.bazel
+load("@rules_pkg//:pkg.bzl", "pkg_zip")
+
+pkg_zip(
+    name = "inet_tcp_proxy_ez",
+    package_dir = "inet_tcp_proxy/ebin",
+    srcs = [
+        "@inet_tcp_proxy//:bazel_erlang_lib",
+    ],
+    package_file_name = "inet_tcp_proxy.ez",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "standard_plugins",
+    srcs = glob(["*.ez"]),
+    visibility = ["//visibility:public"],
+)
+EOF
+"""
+
+http_archive(
+    name = "rabbitmq-server-generic-unix-3.7.28",
+    build_file = "@//:BUILD.package_generic_unix",
+    patch_cmds = [ADD_PLUGINS_DIR_BUILD_FILE],
+    sha256 = "8cc45ef421323b407eda3fa82975fffd81d9a46235f6314e16855caedacd02cc",
+    strip_prefix = "rabbitmq_server-3.7.28",
+    urls = ["https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.7.28/rabbitmq-server-generic-unix-3.7.28.tar.xz"],
+)
diff --git a/deps/rabbit/test/maintenance_mode_SUITE.erl b/deps/rabbit/test/maintenance_mode_SUITE.erl
index 7e6674b58d..db26750b05 100644
--- a/deps/rabbit/test/maintenance_mode_SUITE.erl
+++ b/deps/rabbit/test/maintenance_mode_SUITE.erl
@@ -10,6 +10,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("amqp_client/include/amqp_client.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl").
 
 -compile(export_all).
 
diff --git a/deps/rabbit/test/quorum_queue_SUITE.erl b/deps/rabbit/test/quorum_queue_SUITE.erl
index 571e5ccaff..12950b69e6 100644
--- a/deps/rabbit/test/quorum_queue_SUITE.erl
+++ b/deps/rabbit/test/quorum_queue_SUITE.erl
@@ -184,6 +184,10 @@ init_per_group(Group, Config) ->
     case ClusterSize of
         2 when IsMixed ->
             {skip, "cluster size 2 isn't mixed versions compatible"};
+        3 when IsMixed ->
+            {skip, "cluster size 3 isn't mixed versions compatible"};
+        5 when IsMixed ->
+            {skip, "cluster size 5 isn't mixed versions compatible"};
         _ ->
             Config1 = rabbit_ct_helpers:set_config(Config,
                                                    [{rmq_nodes_count, ClusterSize},
@@ -209,9 +213,12 @@ init_per_group(Group, Config) ->
                             %% tests.
                             timer:sleep(ClusterSize * 1000),
                             Config2;
-                        Skip ->
+                        {skip, _} = Skip ->
                             end_per_group(Group, Config2),
-                            Skip
+                            Skip;
+                        Other ->
+                            end_per_group(Group, Config2),
+                            {skip, Other}
                     end
             end
     end.
@@ -251,9 +258,12 @@ init_per_testcase(Testcase, Config) when Testcase == reconnect_consumer_and_publ
             case EnableFF of
                 ok ->
                     Config3;
-                Skip ->
+                {skip, _} = Skip ->
+                    end_per_testcase(Testcase, Config3),
+                    Skip;
+                Other ->
                     end_per_testcase(Testcase, Config3),
-                    Skip
+                    {skip, Other}
             end
     end;
 init_per_testcase(Testcase, Config) ->
diff --git a/deps/rabbitmq_federation/test/queue_SUITE.erl b/deps/rabbitmq_federation/test/queue_SUITE.erl
index 1c50670e03..dbf091475e 100644
--- a/deps/rabbitmq_federation/test/queue_SUITE.erl
+++ b/deps/rabbitmq_federation/test/queue_SUITE.erl
@@ -88,23 +88,33 @@ init_per_group(classic_queue, Config) ->
        {target_queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}
       ]);
 init_per_group(quorum_queue, Config) ->
-    rabbit_ct_helpers:set_config(
-      Config,
-      [
-       {source_queue_type, quorum},
-       {source_queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]},
-       {target_queue_type, quorum},
-       {target_queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}
-      ]);
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "mixed versions not supported"};
+        _ ->
+            rabbit_ct_helpers:set_config(
+              Config,
+              [
+               {source_queue_type, quorum},
+               {source_queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]},
+               {target_queue_type, quorum},
+               {target_queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}
+              ])
+    end;
 init_per_group(mixed, Config) ->
-    rabbit_ct_helpers:set_config(
-      Config,
-      [
-       {source_queue_type, classic},
-       {source_queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]},
-       {target_queue_type, quorum},
-       {target_queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}
-      ]);
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "mixed versions not supported"};
+        _ ->
+            rabbit_ct_helpers:set_config(
+              Config,
+              [
+               {source_queue_type, classic},
+               {source_queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]},
+               {target_queue_type, quorum},
+               {target_queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}
+              ])
+    end;
 init_per_group(without_disambiguate, Config) ->
     rabbit_ct_helpers:set_config(Config,
       {disambiguate_step, []});
@@ -117,10 +127,15 @@ init_per_group(cluster_size_1 = Group, Config) ->
       ]),
     init_per_group1(Group, Config1);
 init_per_group(cluster_size_2 = Group, Config) ->
-    Config1 = rabbit_ct_helpers:set_config(Config, [
-        {rmq_nodes_count, 2}
-      ]),
-    init_per_group1(Group, Config1).
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "not mixed versions compatible"};
+        _ ->
+            Config1 = rabbit_ct_helpers:set_config(Config, [
+                {rmq_nodes_count, 2}
+            ]),
+            init_per_group1(Group, Config1)
+    end.
 
 init_per_group1(Group, Config) ->
     SetupFederation = case Group of
@@ -142,8 +157,10 @@ init_per_group1(Group, Config) ->
             case rabbit_ct_broker_helpers:enable_feature_flag(Config2, quorum_queue) of
                 ok ->
                     Config2;
-                Skip ->
-                    Skip
+                {skip, _} = Skip ->
+                    Skip;
+                Other ->
+                    {skip, Other}
             end;
         _ ->
             Config2
diff --git a/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl b/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl
index 4c5a29e73c..1c003f7ed3 100644
--- a/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl
+++ b/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl
@@ -91,6 +91,10 @@ end_per_group(_, Config) ->
     Steps = Teardown0 ++ Teardown1,
     rabbit_ct_helpers:run_teardown_steps(Config, Steps).
 
+init_per_testcase(Testcase, Config)
+        when Testcase == is_quorum_critical_test
+            orelse Testcase == is_mirror_sync_critical_test ->
+    {skip, "not mixed versions compatible"};
 init_per_testcase(Testcase, Config) ->
     rabbit_ct_helpers:testcase_started(Config, Testcase).
 
diff --git a/deps/rabbitmq_management_agent/test/rabbit_mgmt_gc_SUITE.erl b/deps/rabbitmq_management_agent/test/rabbit_mgmt_gc_SUITE.erl
index ded7f6757b..8f7f691331 100644
--- a/deps/rabbitmq_management_agent/test/rabbit_mgmt_gc_SUITE.erl
+++ b/deps/rabbitmq_management_agent/test/rabbit_mgmt_gc_SUITE.erl
@@ -78,13 +78,20 @@ end_per_group(_, Config) ->
     Config.
 
 init_per_testcase(quorum_queue_stats = Testcase, Config) ->
-    case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of
-        ok ->
-            rabbit_ct_helpers:testcase_started(Config, Testcase),
-            rabbit_ct_helpers:run_steps(
-              Config, rabbit_ct_client_helpers:setup_steps());
-        Skip ->
-            Skip
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "not mixed versions compatible"};
+        _ ->
+            case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of
+                ok ->
+                    rabbit_ct_helpers:testcase_started(Config, Testcase),
+                    rabbit_ct_helpers:run_steps(
+                    Config, rabbit_ct_client_helpers:setup_steps());
+                {skip, _} = Skip ->
+                    Skip;
+                Other ->
+                    {skip, Other}
+            end
     end;
 init_per_testcase(Testcase, Config) ->
     rabbit_ct_helpers:testcase_started(Config, Testcase),
diff --git a/deps/rabbitmq_peer_discovery_aws/test/integration_SUITE.erl b/deps/rabbitmq_peer_discovery_aws/test/integration_SUITE.erl
index 8b969908e8..6619e04f7e 100644
--- a/deps/rabbitmq_peer_discovery_aws/test/integration_SUITE.erl
+++ b/deps/rabbitmq_peer_discovery_aws/test/integration_SUITE.erl
@@ -42,30 +42,37 @@ groups() ->
     ].
 
 init_per_suite(Config) ->
-    inets:start(),
-    rabbit_ct_helpers:log_environment(),
-    Config1 = rabbit_ct_helpers:set_config(
-                Config, [
-                         {ecs_region, "eu-west-1"},
-                         {ecs_cluster_name, os:getenv("AWS_ECS_CLUSTER_NAME", "rabbitmq-peer-discovery-aws")},
-                         {ecs_profile_name, "rabbitmq-peer-discovery-aws-profile"},
-                         {ecs_instance_role, "ecs-peer-discovery-aws"},
-                         {ecs_cluster_size, ?CLUSTER_SIZE},
-                         {rabbitmq_default_user, "test"},
-                         {rabbitmq_default_pass, rabbit_ct_helpers:random_term_checksum()},
-                         {rabbitmq_erlang_cookie, rabbit_ct_helpers:random_term_checksum()}
-                        ]),
-    Config2 = rabbit_ct_helpers:register_teardown_step(Config1, fun aws_ecs_util:destroy_ecs_cluster/1),
-    rabbit_ct_helpers:run_steps(
-      Config2, [
-                fun rabbit_ct_helpers:init_skip_as_error_flag/1,
-                fun rabbit_ct_helpers:start_long_running_testsuite_monitor/1,
-                fun aws_ecs_util:ensure_aws_cli/1,
-                fun aws_ecs_util:ensure_ecs_cli/1,
-                fun aws_ecs_util:init_aws_credentials/1,
-                fun aws_ecs_util:ensure_rabbitmq_image/1,
-                fun aws_ecs_util:start_ecs_cluster/1
-               ]).
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            %% These test would like passed in mixed versions, but they won't
+            %% actually honor mixed versions as currently specified via env var
+            {skip, "not mixed versions compatible"};
+        _ ->
+            inets:start(),
+            rabbit_ct_helpers:log_environment(),
+            Config1 = rabbit_ct_helpers:set_config(
+                        Config, [
+                                {ecs_region, "eu-west-1"},
+                                {ecs_cluster_name, os:getenv("AWS_ECS_CLUSTER_NAME", "rabbitmq-peer-discovery-aws")},
+                                {ecs_profile_name, "rabbitmq-peer-discovery-aws-profile"},
+                                {ecs_instance_role, "ecs-peer-discovery-aws"},
+                                {ecs_cluster_size, ?CLUSTER_SIZE},
+                                {rabbitmq_default_user, "test"},
+                                {rabbitmq_default_pass, rabbit_ct_helpers:random_term_checksum()},
+                                {rabbitmq_erlang_cookie, rabbit_ct_helpers:random_term_checksum()}
+                                ]),
+            Config2 = rabbit_ct_helpers:register_teardown_step(Config1, fun aws_ecs_util:destroy_ecs_cluster/1),
+            rabbit_ct_helpers:run_steps(
+            Config2, [
+                        fun rabbit_ct_helpers:init_skip_as_error_flag/1,
+                        fun rabbit_ct_helpers:start_long_running_testsuite_monitor/1,
+                        fun aws_ecs_util:ensure_aws_cli/1,
+                        fun aws_ecs_util:ensure_ecs_cli/1,
+                        fun aws_ecs_util:init_aws_credentials/1,
+                        fun aws_ecs_util:ensure_rabbitmq_image/1,
+                        fun aws_ecs_util:start_ecs_cluster/1
+                    ])
+    end.
 
 end_per_suite(Config) ->
     rabbit_ct_helpers:run_teardown_steps(Config).
diff --git a/deps/rabbitmq_shovel/test/dynamic_SUITE.erl b/deps/rabbitmq_shovel/test/dynamic_SUITE.erl
index aa5bf49fcb..2eea7aff36 100644
--- a/deps/rabbitmq_shovel/test/dynamic_SUITE.erl
+++ b/deps/rabbitmq_shovel/test/dynamic_SUITE.erl
@@ -60,9 +60,11 @@ end_per_suite(Config) ->
       rabbit_ct_broker_helpers:teardown_steps()).
 
 init_per_group(quorum_queue_tests, Config) ->
-    case os:getenv("SECONDARY_UMBRELLA") of
-        false -> Config;
-        _     -> {skip, "quorum queue tests are skipped in mixed mode"}
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "quorum queue tests are skipped in mixed mode"};
+        _ ->
+            Config
     end;
 init_per_group(_, Config) ->
     Config.
diff --git a/rabbitmq.bzl b/rabbitmq.bzl
index 2ce2e5ee0c..3a44223e77 100644
--- a/rabbitmq.bzl
+++ b/rabbitmq.bzl
@@ -5,7 +5,7 @@ load(
     "erlang_lib",
     "test_erlang_lib",
 )
-load("@bazel-erlang//:ct_sharded.bzl", "ct_suite")
+load("@bazel-erlang//:ct_sharded.bzl", "ct_suite", "ct_suite_variant")
 load("//:rabbitmq_home.bzl", "rabbitmq_home")
 load("//:rabbitmq_run.bzl", "rabbitmq_run")
 
@@ -129,15 +129,24 @@ def broker_for_integration_suites():
 
 def rabbitmq_integration_suite(
         package,
+        name = None,
+        tags = [],
         data = [],
         erlc_opts = [],
+        additional_hdrs = [],
+        additional_srcs = [],
         test_env = {},
         tools = [],
         deps = [],
         runtime_deps = [],
         **kwargs):
     ct_suite(
+        name = name,
+        suite_name = name,
+        tags = tags,
         erlc_opts = RABBITMQ_TEST_ERLC_OPTS + erlc_opts,
+        additional_hdrs = additional_hdrs,
+        additional_srcs = additional_srcs,
         data = [
             "@rabbitmq_ct_helpers//tools/tls-certs:Makefile",
             "@rabbitmq_ct_helpers//tools/tls-certs:openssl.cnf.in",
@@ -166,7 +175,42 @@ def rabbitmq_integration_suite(
         ] + deps,
         **kwargs
     )
-    return kwargs["name"]
+
+    ct_suite_variant(
+        name = name + "-mixed",
+        suite_name = name,
+        tags = tags + ["mixed-version-cluster"],
+        data = [
+            "@rabbitmq_ct_helpers//tools/tls-certs:Makefile",
+            "@rabbitmq_ct_helpers//tools/tls-certs:openssl.cnf.in",
+        ] + data,
+        test_env = dict({
+            "SKIP_MAKE_TEST_DIST": "true",
+            "RABBITMQ_FEATURE_FLAGS": "",
+            "RABBITMQ_RUN": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/rabbitmq-for-tests-run".format(package),
+            "RABBITMQCTL": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/broker-for-tests-home/sbin/rabbitmqctl".format(package),
+            "RABBITMQ_PLUGINS": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/broker-for-tests-home/sbin/rabbitmq-plugins".format(package),
+            "RABBITMQ_QUEUES": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/broker-for-tests-home/sbin/rabbitmq-queues".format(package),
+            "RABBITMQ_RUN_SECONDARY": "$TEST_SRCDIR/rabbitmq-server-generic-unix-3.7.28/rabbitmq-run",
+        }.items() + test_env.items()),
+        tools = [
+            ":rabbitmq-for-tests-run",
+            "@rabbitmq-server-generic-unix-3.7.28//:rabbitmq-run",
+        ] + tools,
+        runtime_deps = [
+            "//deps/rabbitmq_cli:elixir_as_bazel_erlang_lib",
+            "//deps/rabbitmq_cli:rabbitmqctl",
+            "@rabbitmq_ct_client_helpers//:bazel_erlang_lib",
+        ] + runtime_deps,
+        deps = [
+            "//deps/amqp_client:bazel_erlang_lib",
+            "//deps/rabbit_common:bazel_erlang_lib",
+            "@rabbitmq_ct_helpers//:bazel_erlang_lib",
+        ] + deps,
+        **kwargs
+    )
+
+    return name
 
 def assert_suites(suite_names, suite_files):
     for f in suite_files:
diff --git a/rabbitmq_home.bzl b/rabbitmq_home.bzl
index 3a9101126e..addc65b81b 100644
--- a/rabbitmq_home.bzl
+++ b/rabbitmq_home.bzl
@@ -6,7 +6,6 @@ RabbitmqHomeInfo = provider(
         "sbin": "Files making up the sbin dir",
         "escript": "Files making up the escript dir",
         "plugins": "Files making up the plugins dir",
-        "erlang_version": "Version of the Erlang compiler used",
     },
 )
 
@@ -118,7 +117,6 @@ def _impl(ctx):
             sbin = scripts,
             escript = escripts,
             plugins = plugins,
-            erlang_version = erlang_versions[0],
         ),
         DefaultInfo(
             files = depset(scripts + escripts + plugins),
@@ -140,7 +138,6 @@ rabbitmq_home = rule(
             allow_files = True,
         ),
         "_rabbitmqctl_escript": attr.label(default = "//deps/rabbitmq_cli:rabbitmqctl"),
-        "_erlang_version": attr.label(default = "@bazel-erlang//:erlang_version"),
         "plugins": attr.label_list(),
     },
 )
diff --git a/rabbitmq_package_generic_unix.bzl b/rabbitmq_package_generic_unix.bzl
new file mode 100644
index 0000000000..392c18c8ba
--- /dev/null
+++ b/rabbitmq_package_generic_unix.bzl
@@ -0,0 +1,26 @@
+load("@//:rabbitmq_home.bzl", "RabbitmqHomeInfo")
+
+def _impl(ctx):
+    scripts = ctx.files.sbin
+    escripts = ctx.files.escript
+    plugins = ctx.files.plugins
+
+    return [
+        RabbitmqHomeInfo(
+            sbin = scripts,
+            escript = escripts,
+            plugins = plugins,
+        ),
+        DefaultInfo(
+            files = depset(scripts + escripts + plugins),
+        ),
+    ]
+
+rabbitmq_package_generic_unix = rule(
+    implementation = _impl,
+    attrs = {
+        "sbin": attr.label_list(allow_files = True),
+        "escript": attr.label_list(allow_files = True),
+        "plugins": attr.label_list(allow_files = True),
+    },
+)
diff --git a/rabbitmq_run.bzl b/rabbitmq_run.bzl
index 4bb7c77c56..f01fef395c 100644
--- a/rabbitmq_run.bzl
+++ b/rabbitmq_run.bzl
@@ -3,38 +3,34 @@ load("@bazel-erlang//:bazel_erlang_lib.bzl", "path_join")
 load("@bazel-erlang//:ct.bzl", "sanitize_sname")
 load(":rabbitmq_home.bzl", "RabbitmqHomeInfo")
 
-# Note: Theses rules take advantage of the fact that when the files from
-#       the rabbitmq_home rule are used as runfiles, they are linked in
-#       at their declared relative paths. In other words, since
-#       rabbitmq_home declares "sbin/rabbitmq-server", is still at
-#       "sbin/rabbitmq-server" when our script runs.
+def _dirname(p):
+    return p.rpartition("/")[0]
+
+def _rabbitmq_home_info_root_short_path(rabbitmq_home):
+    return _dirname(_dirname(rabbitmq_home.sbin[0].short_path))
 
 def _impl(ctx):
-    erlang_version = ctx.attr._erlang_version[ErlangVersionProvider].version
     rabbitmq_home = ctx.attr.home[RabbitmqHomeInfo]
 
-    if rabbitmq_home.erlang_version != erlang_version:
-        fail("Mismatched erlang versions", erlang_version, rabbitmq_home.erlang_version)
+    root = _rabbitmq_home_info_root_short_path(rabbitmq_home)
 
-    erl_libs = ":".join(
-        [p.short_path for p in rabbitmq_home.plugins],
-    )
+    erl_libs = [path_join(root, "plugins")]
 
     ctx.actions.expand_template(
         template = ctx.file._template,
         output = ctx.outputs.executable,
         substitutions = {
-            "{RABBITMQ_HOME}": ctx.attr.home.label.name,
-            "{ERL_LIBS}": erl_libs,
+            "{RABBITMQ_HOME}": root,
+            "{ERL_LIBS}": ":".join(erl_libs),
             "{ERLANG_HOME}": ctx.attr._erlang_home[ErlangHomeProvider].path,
             "{SNAME}": sanitize_sname("sbb-" + ctx.attr.name),
         },
         is_executable = True,
     )
 
-    return [DefaultInfo(
-        runfiles = ctx.runfiles(ctx.attr.home[DefaultInfo].files.to_list()),
-    )]
+    runfiles = ctx.runfiles(ctx.attr.home[DefaultInfo].files.to_list())
+
+    return [DefaultInfo(runfiles = runfiles)]
 
 rabbitmq_run = rule(
     implementation = _impl,
@@ -44,7 +40,6 @@ rabbitmq_run = rule(
             allow_single_file = True,
         ),
         "_erlang_home": attr.label(default = "@bazel-erlang//:erlang_home"),
-        "_erlang_version": attr.label(default = "@bazel-erlang//:erlang_version"),
         "home": attr.label(providers = [RabbitmqHomeInfo]),
     },
     executable = True,
diff --git a/rabbitmqctl.bzl b/rabbitmqctl.bzl
index ccf430225b..da01a58ea0 100644
--- a/rabbitmqctl.bzl
+++ b/rabbitmqctl.bzl
@@ -6,9 +6,6 @@ def _impl(ctx):
 
     rabbitmq_home = ctx.attr.home[RabbitmqHomeInfo]
 
-    if rabbitmq_home.erlang_version != erlang_version:
-        fail("Mismatched erlang versions", erlang_version, rabbitmq_home.erlang_version)
-
     script = """
     exec ./{}/sbin/rabbitmqctl $@
     """.format(ctx.attr.home.label.name)
diff --git a/scripts/bazel/rabbitmq-run.sh b/scripts/bazel/rabbitmq-run.sh
index 94e0ada981..91e1c77a40 100644
--- a/scripts/bazel/rabbitmq-run.sh
+++ b/scripts/bazel/rabbitmq-run.sh
@@ -1,8 +1,11 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# https://stackoverflow.com/a/4774063
-SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+if [ -z ${TEST_SRCDIR+x} ]; then
+BASE_DIR=$PWD
+else
+BASE_DIR=$TEST_SRCDIR/$TEST_WORKSPACE
+fi
 
 if [ $1 = "-C" ]; then
     cd $2
@@ -32,13 +35,13 @@ for arg in "$@"; do
     esac
 done
 
-DEFAULT_PLUGINS_DIR=${SCRIPTPATH}/{RABBITMQ_HOME}/plugins
+DEFAULT_PLUGINS_DIR=${BASE_DIR}/{RABBITMQ_HOME}/plugins
 if [ ! -z ${EXTRA_PLUGINS_DIR+x} ]; then
     DEFAULT_PLUGINS_DIR=${DEFAULT_PLUGINS_DIR}:${EXTRA_PLUGINS_DIR}
 fi
 
 TEST_TMPDIR=${TEST_TMPDIR:=${TMPDIR}/rabbitmq-test-instances}
-RABBITMQ_SCRIPTS_DIR=${SCRIPTPATH}/{RABBITMQ_HOME}/sbin
+RABBITMQ_SCRIPTS_DIR=${BASE_DIR}/{RABBITMQ_HOME}/sbin
 RABBITMQ_PLUGINS=${RABBITMQ_SCRIPTS_DIR}/rabbitmq-plugins
 RABBITMQ_SERVER=${RABBITMQ_SCRIPTS_DIR}/rabbitmq-server
 RABBITMQCTL=${RABBITMQ_SCRIPTS_DIR}/rabbitmqzctl
@@ -173,12 +176,12 @@ case $CMD in
             while ps -p "$pid" >/dev/null 2>&1; do sleep 1; done
         ;;
     set-resource-alarm)
-        ERL_LIBS="{ERL_LIBS}" \
+        ERL_LIBS="${BASE_DIR}/{ERL_LIBS}" \
            ${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl -n ${RABBITMQ_NODENAME} \
             eval "rabbit_alarm:set_alarm({{resource_limit, ${SOURCE}, node()}, []})."
         ;;
     clear-resource-alarm)
-        ERL_LIBS="{ERL_LIBS}" \
+        ERL_LIBS="${BASE_DIR}/{ERL_LIBS}" \
             ${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl -n ${RABBITMQ_NODENAME} \
             eval "rabbit_alarm:clear_alarm({resource_limit, ${SOURCE}, node()})."
         ;;