From 18fd4bde03891322dd3a96c20573dbac9be1271d Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 17:01:17 +0200 Subject: rabbitmq_cli dialyze enhancements (backport #8148) (backport #8151) (#8152) * Build cli deps as .ez archives This provides an elixir/erlang agnostic way of providing them other erlang rules (cherry picked from commit 19f4abd55b0d25b61feb75ff4447b75490fe0700) (cherry picked from commit 166b2fdfa8ccb4dedd6e97ad4203b2a4c8ad2aae) * Pass csv and json to rabbitmq_cli plt So that they are no longer reported as unknown in //deps/rabbitmq_cli:dialyze (cherry picked from commit 08061144ad32aa829b92ed0f4afb7273535c99b0) (cherry picked from commit 5e45f529f3809ca22f88a3ae7be0e385acb26cd0) * Account for Elixir containing several core applications - eex - elixir - ex_unit - iex - logger - mix So that apps (like rabbitmq_cli) can dialyze against the extra components (cherry picked from commit ea895a00230874b0971eca18a735ad8826896bc1) (cherry picked from commit c14a9ebcb3ece058d21433186463894656f0d220) * Use rules_erlang 3.10.0 (cherry picked from commit 92b0a81c609c3a774028cb54953084ee0bda1fe6) (cherry picked from commit 16c7a9b9f44b12fadc6ce9e32d416d15a5466f69) # Conflicts: # MODULE.bazel * Fix github actions cache key for the bazel repo-cache (cherry picked from commit 204d0177c25b6a16cf9ba6923d61df165bc85b07) (cherry picked from commit 27d4a40e3b4c710b5ce91aa63418b2036278d3a6) * Update rules_pkg to 0.9.1 (cherry picked from commit dad871e86c6bece4a09a02cc84dda231b2bcbc11) (cherry picked from commit 9a747c972eeeeccc71b7dab15461ba2718806850) * resolve merge conflicts --------- Co-authored-by: Rin Kuryloski --- .github/workflows/test-mixed-versions.yaml | 8 +++- .github/workflows/test-selenium.yaml | 4 +- .github/workflows/test.yaml | 8 +++- BUILD.bazel | 2 +- MODULE.bazel | 28 +++++++------- WORKSPACE | 15 +++++++- bazel/elixir/BUILD.bazel | 12 ++++++ bazel/elixir/elixir_as_app.bzl | 14 +++++-- bazel/elixir/mix_archive_build.bzl | 60 ++++++++++++++++++------------ deps/rabbitmq_cli/BUILD.bazel | 34 +++++++++++++++++ deps/rabbitmq_cli/rabbitmqctl.bzl | 2 +- deps/rabbitmq_cli/rabbitmqctl_test.bzl | 4 +- dist.bzl | 2 +- 13 files changed, 142 insertions(+), 51 deletions(-) diff --git a/.github/workflows/test-mixed-versions.yaml b/.github/workflows/test-mixed-versions.yaml index 8d56ca8816..9ab6d7441f 100644 --- a/.github/workflows/test-mixed-versions.yaml +++ b/.github/workflows/test-mixed-versions.yaml @@ -135,7 +135,9 @@ jobs: uses: actions/cache@v3.3.1 with: path: "/home/runner/repo-cache/" - key: repo-cache + key: ${{ runner.os }}-repo-cache-${{ hashFiles('MODULE.bazel','WORKSPACE','bazel/bzlmod/secondary_umbrella.bzl') }} + restore-keys: | + ${{ runner.os }}-repo-cache- - name: CONFIGURE BAZEL run: | if [ -n "${{ secrets.BUILDBUDDY_API_KEY }}" ]; then @@ -183,7 +185,9 @@ jobs: uses: actions/cache@v3.3.1 with: path: "/home/runner/repo-cache/" - key: repo-cache + key: ${{ runner.os }}-repo-cache-${{ hashFiles('MODULE.bazel','WORKSPACE','bazel/bzlmod/secondary_umbrella.bzl') }} + restore-keys: | + ${{ runner.os }}-repo-cache- - name: CONFIGURE BAZEL run: | ERLANG_HOME="$(dirname $(dirname $(which erl)))" diff --git a/.github/workflows/test-selenium.yaml b/.github/workflows/test-selenium.yaml index 19c38b6ee5..971fc1e78f 100644 --- a/.github/workflows/test-selenium.yaml +++ b/.github/workflows/test-selenium.yaml @@ -50,7 +50,9 @@ jobs: uses: actions/cache@v3.3.1 with: path: "/home/runner/repo-cache/" - key: repo-cache + key: ${{ runner.os }}-repo-cache-${{ hashFiles('MODULE.bazel','WORKSPACE','bazel/bzlmod/secondary_umbrella.bzl') }} + restore-keys: | + ${{ runner.os }}-repo-cache- - name: Configure Bazel run: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9aceb026e7..10ddf4cf04 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -44,7 +44,9 @@ jobs: uses: actions/cache@v3.3.1 with: path: "/home/runner/repo-cache/" - key: repo-cache + key: ${{ runner.os }}-repo-cache-${{ hashFiles('MODULE.bazel','WORKSPACE','bazel/bzlmod/secondary_umbrella.bzl') }} + restore-keys: | + ${{ runner.os }}-repo-cache- - name: CONFIGURE BAZEL run: | if [ -n "${{ secrets.BUILDBUDDY_API_KEY }}" ]; then @@ -96,7 +98,9 @@ jobs: uses: actions/cache@v3.3.1 with: path: "/home/runner/repo-cache/" - key: repo-cache + key: ${{ runner.os }}-repo-cache-${{ hashFiles('MODULE.bazel','WORKSPACE','bazel/bzlmod/secondary_umbrella.bzl') }} + restore-keys: | + ${{ runner.os }}-repo-cache- - name: CONFIGURE BAZEL run: | ERLANG_HOME="$(dirname $(dirname $(which erl)))" diff --git a/BUILD.bazel b/BUILD.bazel index 432a16c8b4..f5f8f43807 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -3,7 +3,7 @@ load( "bool_flag", "string_flag", ) -load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("@rules_pkg//pkg:mappings.bzl", "pkg_files") load("@bazel_gazelle//:def.bzl", "gazelle") load("@rules_erlang//gazelle:def.bzl", "GAZELLE_ERLANG_RUNTIME_DEPS") load("@rules_erlang//:erlang_bytecode2.bzl", "erlc_opts") diff --git a/MODULE.bazel b/MODULE.bazel index 0fe012fecb..3ababaef64 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -5,7 +5,7 @@ module( bazel_dep( name = "rules_pkg", - version = "0.5.1", + version = "0.9.1", ) bazel_dep( @@ -31,7 +31,7 @@ bazel_dep( bazel_dep( name = "rules_erlang", - version = "3.9.14", + version = "3.10.0", ) erlang_config = use_extension( @@ -104,18 +104,18 @@ use_repo( ) register_toolchains( - "@erlang_config//24:toolchain", - "@erlang_config//24:toolchain2", - "@erlang_config//25_0:toolchain", - "@erlang_config//25_0:toolchain2", - "@erlang_config//25_1:toolchain", - "@erlang_config//25_1:toolchain2", - "@erlang_config//25_2:toolchain", - "@erlang_config//25_2:toolchain2", - "@erlang_config//25_3:toolchain", - "@erlang_config//25_3:toolchain2", - "@erlang_config//git_master:toolchain", - "@erlang_config//git_master:toolchain2", + "@erlang_config//24:toolchain_major", + "@erlang_config//24:toolchain_major_minor", + "@erlang_config//25_0:toolchain_major", + "@erlang_config//25_0:toolchain_major_minor", + "@erlang_config//25_1:toolchain_major", + "@erlang_config//25_1:toolchain_major_minor", + "@erlang_config//25_2:toolchain_major", + "@erlang_config//25_2:toolchain_major_minor", + "@erlang_config//25_3:toolchain_major", + "@erlang_config//25_3:toolchain_major_minor", + "@erlang_config//git_master:toolchain_major", + "@erlang_config//git_master:toolchain_major_minor", "@elixir_config//external:toolchain", "@elixir_config//1_13:toolchain", "@elixir_config//1_14:toolchain", diff --git a/WORKSPACE b/WORKSPACE index fff09f2692..b6ac3fdda5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,10 +3,23 @@ workspace(name = "rabbitmq-server") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") +http_archive( + name = "rules_pkg", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", + ], + sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8", +) + +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") + +rules_pkg_dependencies() + git_repository( name = "rules_erlang", remote = "https://github.com/rabbitmq/rules_erlang.git", - tag = "3.9.14", + tag = "3.10.0", ) load("@rules_erlang//:internal_deps.bzl", "rules_erlang_internal_deps") diff --git a/bazel/elixir/BUILD.bazel b/bazel/elixir/BUILD.bazel index b362f11b13..f21e9181d4 100644 --- a/bazel/elixir/BUILD.bazel +++ b/bazel/elixir/BUILD.bazel @@ -12,3 +12,15 @@ elixir_as_app( name = "erlang_app", visibility = ["//visibility:public"], ) + +elixir_as_app( + name = "logger", + app = "logger", + visibility = ["//visibility:public"], +) + +elixir_as_app( + name = "iex", + app = "iex", + visibility = ["//visibility:public"], +) diff --git a/bazel/elixir/elixir_as_app.bzl b/bazel/elixir/elixir_as_app.bzl index b653a8dcdd..eb615722c9 100644 --- a/bazel/elixir/elixir_as_app.bzl +++ b/bazel/elixir/elixir_as_app.bzl @@ -2,13 +2,17 @@ load( "@rules_erlang//:erlang_app_info.bzl", "ErlangAppInfo", ) +load( + "@rules_erlang//:util.bzl", + "path_join", +) load( ":elixir_toolchain.bzl", "elixir_dirs", ) def _impl(ctx): - ebin = ctx.actions.declare_directory("ebin") + ebin = ctx.actions.declare_directory(path_join(ctx.label.name, "ebin")) (elixir_home, elixir_runfiles) = elixir_dirs(ctx) @@ -17,9 +21,10 @@ def _impl(ctx): outputs = [ebin], command = """set -euo pipefail -cp -r "{elixir_home}"/lib/elixir/ebin/* {ebin} +cp -r "{elixir_home}"/lib/{app}/ebin/* {ebin} """.format( elixir_home = elixir_home, + app = ctx.attr.app, ebin = ebin.path, ), ) @@ -27,7 +32,7 @@ cp -r "{elixir_home}"/lib/elixir/ebin/* {ebin} return [ DefaultInfo(files = depset([ebin])), ErlangAppInfo( - app_name = "elixir", + app_name = ctx.attr.app, include = [], beam = [ebin], priv = [], @@ -39,6 +44,9 @@ cp -r "{elixir_home}"/lib/elixir/ebin/* {ebin} elixir_as_app = rule( implementation = _impl, + attrs = { + "app": attr.string(default = "elixir"), + }, toolchains = [":toolchain_type"], provides = [ErlangAppInfo], ) diff --git a/bazel/elixir/mix_archive_build.bzl b/bazel/elixir/mix_archive_build.bzl index 9652e0f1de..adc3b15de6 100644 --- a/bazel/elixir/mix_archive_build.bzl +++ b/bazel/elixir/mix_archive_build.bzl @@ -1,7 +1,4 @@ -load( - "@rules_erlang//:util.bzl", - "path_join", -) +load("@bazel_skylib//lib:shell.bzl", "shell") load( "@rules_erlang//private:util.bzl", "additional_file_dest_relative_path", @@ -20,24 +17,20 @@ def _impl(ctx): out = ctx.actions.declare_file(ctx.attr.out.name) mix_invocation_dir = ctx.actions.declare_directory("{}_mix".format(ctx.label.name)) - package_dir = path_join( - ctx.label.workspace_root, - ctx.label.package, - ) - copy_srcs_commands = [] - for src in ctx.files.srcs: - dest = additional_file_dest_relative_path(ctx.label, src) - copy_srcs_commands.extend([ - 'mkdir -p "$(dirname ${{MIX_INVOCATION_DIR}}/{dest})"'.format( - dest = dest, - ), - 'cp {flags}"{src}" "${{MIX_INVOCATION_DIR}}/{dest}"'.format( - flags = "-r " if src.is_directory else "", - src = src.path, - dest = dest, - ), - ]) + for src in ctx.attr.srcs: + for src_file in src[DefaultInfo].files.to_list(): + dest = additional_file_dest_relative_path(src.label, src_file) + copy_srcs_commands.extend([ + 'mkdir -p "$(dirname ${{MIX_INVOCATION_DIR}}/{dest})"'.format( + dest = dest, + ), + 'cp {flags}"{src}" "${{MIX_INVOCATION_DIR}}/{dest}"'.format( + flags = "-r " if src_file.is_directory else "", + src = src_file.path, + dest = dest, + ), + ]) script = """set -euo pipefail @@ -60,11 +53,23 @@ MIX_INVOCATION_DIR="{mix_invocation_dir}" {copy_srcs_commands} +ORIGINAL_DIR=$PWD cd "${{MIX_INVOCATION_DIR}}" export HOME="${{PWD}}" export MIX_ENV=prod export ERL_COMPILER_OPTIONS=deterministic -"${{ABS_ELIXIR_HOME}}"/bin/mix archive.build -o "${{ABS_OUT_PATH}}" +for archive in {archives}; do + "${{ABS_ELIXIR_HOME}}"/bin/mix archive.install --force $ORIGINAL_DIR/$archive +done +if [[ -n "{ez_deps}" ]]; then + mkdir -p _build/${{MIX_ENV}}/lib + for ez_dep in {ez_deps}; do + unzip -q $ORIGINAL_DIR/$ez_dep -d _build/${{MIX_ENV}}/lib + done +fi +"${{ABS_ELIXIR_HOME}}"/bin/mix archive.build \\ + --no-deps-check \\ + -o "${{ABS_OUT_PATH}}" # remove symlinks from the _build directory since it # is an unused output, and bazel does not allow them @@ -75,7 +80,8 @@ find . -type l -delete elixir_home = elixir_home, mix_invocation_dir = mix_invocation_dir.path, copy_srcs_commands = "\n".join(copy_srcs_commands), - package_dir = package_dir, + archives = " ".join([shell.quote(a.path) for a in ctx.files.archives]), + ez_deps = " ".join([shell.quote(a.path) for a in ctx.files.ez_deps]), out = out.path, ) @@ -84,6 +90,8 @@ find . -type l -delete transitive = [ erlang_runfiles.files, elixir_runfiles.files, + depset(ctx.files.archives), + depset(ctx.files.ez_deps), ], ) @@ -110,6 +118,12 @@ mix_archive_build = rule( mandatory = True, allow_files = True, ), + "archives": attr.label_list( + allow_files = [".ez"], + ), + "ez_deps": attr.label_list( + allow_files = [".ez"], + ), "out": attr.output(), }, toolchains = [ diff --git a/deps/rabbitmq_cli/BUILD.bazel b/deps/rabbitmq_cli/BUILD.bazel index b2dcc1c3e0..cf212f27c0 100644 --- a/deps/rabbitmq_cli/BUILD.bazel +++ b/deps/rabbitmq_cli/BUILD.bazel @@ -5,6 +5,34 @@ load(":rabbitmqctl_test.bzl", "rabbitmqctl_test") load("//:rabbitmq_home.bzl", "rabbitmq_home") load("//:rabbitmq_run.bzl", "rabbitmq_run") load("//:rabbitmq.bzl", "RABBITMQ_DIALYZER_OPTS", "STARTS_BACKGROUND_BROKER_TAG", "without") +load( + "@rabbitmq-server//bazel/elixir:mix_archive_build.bzl", + "mix_archive_build", +) + +mix_archive_build( + name = "parallel_stream_ez", + srcs = ["@parallel_stream//:sources"], + archives = ["@hex//:archive"], + out = "parallel_stream.ez", +) + +mix_archive_build( + name = "csv_ez", + srcs = ["@csv//:sources"], + archives = ["@hex//:archive"], + ez_deps = [ + ":parallel_stream_ez", + ], + out = "csv.ez", +) + +mix_archive_build( + name = "json_ez", + srcs = ["@json//:sources"], + archives = ["@hex//:archive"], + out = "json.ez", +) # Note: All the various rabbitmq-* scripts are just copies of rabbitmqctl rabbitmqctl( @@ -88,9 +116,15 @@ plt( libs = [":elixir"], deps = [ ":elixir", + "//bazel/elixir:logger", "//deps/rabbit:erlang_app", "//deps/rabbit_common:erlang_app", ], + ez_deps = [ + ":parallel_stream_ez", + ":csv_ez", + ":json_ez", + ], ) dialyze( diff --git a/deps/rabbitmq_cli/rabbitmqctl.bzl b/deps/rabbitmq_cli/rabbitmqctl.bzl index 35f330f9d9..10890d3457 100644 --- a/deps/rabbitmq_cli/rabbitmqctl.bzl +++ b/deps/rabbitmq_cli/rabbitmqctl.bzl @@ -149,7 +149,7 @@ find . -type l -delete escript_path = escript.path, ebin_dir = ebin.path, consolidated_dir = consolidated.path, - archives = "".join([shell.quote(a.path) for a in ctx.files.archives]), + archives = " ".join([shell.quote(a.path) for a in ctx.files.archives]), precompiled_deps = " ".join([ dep[ErlangAppInfo].app_name for dep in ctx.attr.deps diff --git a/deps/rabbitmq_cli/rabbitmqctl_test.bzl b/deps/rabbitmq_cli/rabbitmqctl_test.bzl index f3ef9f5169..3ba076f2f9 100644 --- a/deps/rabbitmq_cli/rabbitmqctl_test.bzl +++ b/deps/rabbitmq_cli/rabbitmqctl_test.bzl @@ -139,7 +139,7 @@ set -x elixir_home = elixir_home, package_dir = package_dir, deps_dir = deps_dir, - archives = "".join([shell.quote(a.short_path) for a in ctx.files.archives]), + archives = " ".join([shell.quote(a.short_path) for a in ctx.files.archives]), precompiled_deps = precompiled_deps, rabbitmq_run_cmd = ctx.attr.rabbitmq_run[DefaultInfo].files_to_run.executable.short_path, ) @@ -196,7 +196,7 @@ exit /b 1 elixir_home = windows_path(elixir_home), package_dir = windows_path(ctx.label.package), deps_dir = deps_dir, - archives = "".join([shell.quote(a.short_path) for a in ctx.files.archives]), + archives = " ".join([shell.quote(a.short_path) for a in ctx.files.archives]), precompiled_deps = precompiled_deps, rabbitmq_run_cmd = ctx.attr.rabbitmq_run[DefaultInfo].files_to_run.executable.short_path, ) diff --git a/dist.bzl b/dist.bzl index 9bed69d84a..b8d5380f89 100644 --- a/dist.bzl +++ b/dist.bzl @@ -1,4 +1,4 @@ -load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", "pkg_mkdirs", "strip_prefix") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files") load("@rules_pkg//:pkg.bzl", "pkg_tar") load("@rules_erlang//:erlang_app_info.bzl", "ErlangAppInfo", "flat_deps") load("@rules_erlang//:util.bzl", "path_join") -- cgit v1.2.1