diff options
author | dcorbacho <dparracorbacho@piotal.io> | 2021-11-29 10:12:30 +0100 |
---|---|---|
committer | dcorbacho <dparracorbacho@piotal.io> | 2021-11-29 10:12:30 +0100 |
commit | a7af1f815fa8db7f96cb75dd152ae729deb0dc6b (patch) | |
tree | 3fd2733ac20e368b98a232de4aef019a4ab2d790 | |
parent | dcfa342286f83b47bd2f1394b7cb0a69dbb6bfbd (diff) | |
parent | 83126f12851c499eb10c955b5e622446f1ddcdf3 (diff) | |
download | rabbitmq-server-git-a7af1f815fa8db7f96cb75dd152ae729deb0dc6b.tar.gz |
Merge branch 'master' of github.com:rabbitmq/rabbitmq-server
29 files changed, 947 insertions, 163 deletions
diff --git a/.github/workflows/update-rbe-images.yaml b/.github/workflows/update-rbe-images.yaml index c2174fedbd..a988b66a2f 100644 --- a/.github/workflows/update-rbe-images.yaml +++ b/.github/workflows/update-rbe-images.yaml @@ -29,7 +29,7 @@ jobs: with: path: rbe-erlang-platform repository: rabbitmq/rbe-erlang-platform - branch: linux-erlang-${{ matrix.erlang_version }} + ref: linux-erlang-${{ matrix.erlang_version }} - name: DETERMINE LATEST COMMIT id: find-commit working-directory: rbe-erlang-platform diff --git a/BUILD.bazel b/BUILD.bazel index 9ea70fc6e3..771f0f259c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,9 +1,12 @@ +load("@rules_pkg//:pkg.bzl", "pkg_tar") load("@bazel-erlang//:dialyze.bzl", "plt") load("@bazel-erlang//:shell.bzl", "shell") load("elixir_home.bzl", "elixir_home") load(":rabbitmq_home.bzl", "rabbitmq_home") load(":rabbitmq_run.bzl", "rabbitmq_run", "rabbitmq_run_command") load(":rabbitmqctl.bzl", "rabbitmqctl") +load(":rabbitmq.bzl", "ALL_PLUGINS", "APP_VERSION") +load(":dist.bzl", "collect_licenses", "versioned_rabbitmq_home") exports_files([ "scripts/bazel/rabbitmq-run.sh", @@ -37,42 +40,6 @@ plt( visibility = ["//visibility:public"], ) -ALL_PLUGINS = [ - "//deps/rabbit:bazel_erlang_lib", - "//deps/rabbitmq_amqp1_0:bazel_erlang_lib", - "//deps/rabbitmq_auth_backend_cache:bazel_erlang_lib", - "//deps/rabbitmq_auth_backend_http:bazel_erlang_lib", - "//deps/rabbitmq_auth_backend_ldap:bazel_erlang_lib", - "//deps/rabbitmq_auth_backend_oauth2:bazel_erlang_lib", - "//deps/rabbitmq_auth_mechanism_ssl:bazel_erlang_lib", - "//deps/rabbitmq_consistent_hash_exchange:bazel_erlang_lib", - "//deps/rabbitmq_event_exchange:bazel_erlang_lib", - "//deps/rabbitmq_federation:bazel_erlang_lib", - "//deps/rabbitmq_federation_management:bazel_erlang_lib", - "//deps/rabbitmq_jms_topic_exchange:bazel_erlang_lib", - "//deps/rabbitmq_management:bazel_erlang_lib", - "//deps/rabbitmq_mqtt:bazel_erlang_lib", - "//deps/rabbitmq_peer_discovery_aws:bazel_erlang_lib", - "//deps/rabbitmq_peer_discovery_consul:bazel_erlang_lib", - "//deps/rabbitmq_peer_discovery_etcd:bazel_erlang_lib", - "//deps/rabbitmq_peer_discovery_k8s:bazel_erlang_lib", - "//deps/rabbitmq_prometheus:bazel_erlang_lib", - "//deps/rabbitmq_random_exchange:bazel_erlang_lib", - "//deps/rabbitmq_recent_history_exchange:bazel_erlang_lib", - "//deps/rabbitmq_sharding:bazel_erlang_lib", - "//deps/rabbitmq_shovel:bazel_erlang_lib", - "//deps/rabbitmq_shovel_management:bazel_erlang_lib", - "//deps/rabbitmq_stomp:bazel_erlang_lib", - "//deps/rabbitmq_stream:bazel_erlang_lib", - "//deps/rabbitmq_stream_management:bazel_erlang_lib", - "//deps/rabbitmq_top:bazel_erlang_lib", - "//deps/rabbitmq_tracing:bazel_erlang_lib", - "//deps/rabbitmq_trust_store:bazel_erlang_lib", - "//deps/rabbitmq_web_dispatch:bazel_erlang_lib", - "//deps/rabbitmq_web_mqtt:bazel_erlang_lib", - "//deps/rabbitmq_web_stomp:bazel_erlang_lib", -] - rabbitmq_home( name = "broker-home", plugins = ALL_PLUGINS, @@ -132,3 +99,68 @@ shell( name = "repl", deps = ALL_PLUGINS, ) + +collect_licenses( + name = "licenses", + srcs = glob( + ["LICENSE*"], + exclude = [ + "LICENSE.md", + "LICENSE.txt", + ], + ), + deps = ALL_PLUGINS, +) + +versioned_rabbitmq_home( + name = "dist-home", + plugins = ALL_PLUGINS, +) + +pkg_tar( + name = "license-files", + srcs = [ + ":licenses", + "//deps/rabbit:INSTALL", + ], + visibility = ["//visibility:public"], +) + +pkg_tar( + name = "scripts", + srcs = [ + "scripts/bash_autocomplete.sh", + "scripts/rabbitmq-script-wrapper", + "scripts/rabbitmqctl-autocomplete.sh", + "scripts/zsh_autocomplete.sh", + ], + package_dir = "scripts", + visibility = ["//visibility:public"], +) + +pkg_tar( + name = "release-notes", + srcs = glob([ + "release-notes/*.md", + "release-notes/*.txt", + ]), + package_dir = "release-notes", + visibility = ["//visibility:public"], +) + +pkg_tar( + name = "package-generic-unix", + srcs = [ + ":dist-home", + ], + extension = "tar.xz", + package_dir = "rabbitmq_server-{}".format(APP_VERSION), + strip_prefix = "dist-home", + visibility = ["//visibility:public"], + deps = [ + ":license-files", + ":release-notes", + ":scripts", + "//deps/rabbit:manpages-dir", + ], +) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index fa183dbd5d..b6c0d665f4 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -2,6 +2,19 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") http_archive( + name = "rules_pkg", + sha256 = "038f1caa773a7e35b3663865ffb003169c6a71dc995e39bf4815792f385d837d", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz", + ], +) + +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") + +rules_pkg_dependencies() + +http_archive( name = "io_buildbuddy_buildbuddy_toolchain", sha256 = "48546946879b1fd2dcba327ba15776c822f2ce9a9ef1077be9bf3ecadcc1564a", strip_prefix = "buildbuddy-toolchain-b2f5e7e3b126c6d7cf243227147478c0959bfc95", @@ -21,13 +34,13 @@ buildbuddy( git_repository( name = "rbe_23", - commit = "13b3045476b02aaaa2c91030fdaff91e7ab9bced", + commit = "d2b454dc5138a2a92de45a0a672241a4fbb5a1e5", remote = "https://github.com/rabbitmq/rbe-erlang-platform.git", ) git_repository( name = "rbe_24", - commit = "41a52ed360537f6794f6e759dd7eec7bbf65445c", + commit = "a087892ef4202dc3245b64d36d5921491848315f", remote = "https://github.com/rabbitmq/rbe-erlang-platform.git", ) @@ -46,7 +59,7 @@ rules_pkg_dependencies() git_repository( name = "bazel-erlang", - commit = "63f7f4213a4f50e2bd891c98be403a45def8d1cf", + commit = "050faedb2a3422a60d6b98678c714ed1a61ec71d", remote = "https://github.com/rabbitmq/bazel-erlang.git", ) diff --git a/deps/rabbit/BUILD.bazel b/deps/rabbit/BUILD.bazel index 01ec856b50..01be7ae4e2 100644 --- a/deps/rabbit/BUILD.bazel +++ b/deps/rabbit/BUILD.bazel @@ -23,6 +23,7 @@ exports_files([ "scripts/rabbitmq-queues", "scripts/rabbitmq-server", "scripts/rabbitmqctl", + "INSTALL", ]) _APP_ENV = """[ @@ -997,3 +998,72 @@ assert_suites( suites, glob(["test/**/*_SUITE.erl"]), ) + +filegroup( + name = "manpages", + srcs = glob([ + "docs/*.1", + "docs/*.2", + "docs/*.3", + "docs/*.4", + "docs/*.5", + "docs/*.6", + "docs/*.7", + "docs/*.8", + "docs/*.9", + ]), +) + +genrule( + name = "manpages-dir", + srcs = [":manpages"], + outs = ["manpages.tar"], + cmd = """set -euo pipefail + +DESTDIR=manpages-tmp/share/man +mkdir -p $${DESTDIR} +for mp in $(SRCS); do + section=$${mp##*.} + mkdir -p $${DESTDIR}/man$$section + gzip < $$mp \\ + > $${DESTDIR}/man$$section/$$(basename $$mp).gz +done +tar --strip-components 1 -cf $@ manpages-tmp/* +rm -dr manpages-tmp +""", + visibility = ["//visibility:public"], +) + +genrule( + name = "web-manpages", + srcs = [":manpages"], + outs = ["web-manpages.tar"], + cmd = """set -euo pipefail + +mkdir web-manpages-tmp +for mp in $(SRCS); do + d=web-manpages-tmp/$$(basename $${mp}).html + echo "Converting $$mp to $$d..." + mandoc -T html -O 'fragment,man=%N.%S.html' "$$mp" | \\ + awk '\\ + /^<table class="head">$$/ { remove_table=1; next; } \\ + /^<table class="foot">$$/ { remove_table=1; next; } \\ + /^<\\/table>$$/ { if (remove_table) { remove_table=0; next; } } \\ + { if (!remove_table) { \\ + line=$$0; \\ + gsub(/<h2/, "<h3", line); \\ + gsub(/<\\/h2>/, "</h3>", line); \\ + gsub(/<h1/, "<h2", line); \\ + gsub(/<\\/h1>/, "</h2>", line); \\ + gsub(/class="D1"/, "class=\"D1 lang-bash\"", line); \\ + gsub(/class="Bd Bd-indent"/, "class=\"Bd Bd-indent lang-bash\"", line); \\ + gsub(/&#[xX]201[cCdD];/, "\\"", line); \\ + print line; \\ + } } \\ + ' > "$$d" +done +tar --strip-components 1 -cf $@ web-manpages-tmp/* +rm -dr web-manpages-tmp +""", + visibility = ["//visibility:public"], +) diff --git a/deps/rabbit/docs/rabbitmq-server.service.example b/deps/rabbit/docs/rabbitmq-server.service.example index dec70eb635..69531b1ff6 100644 --- a/deps/rabbit/docs/rabbitmq-server.service.example +++ b/deps/rabbit/docs/rabbitmq-server.service.example @@ -5,6 +5,19 @@ After=network.target epmd@0.0.0.0.socket Wants=network.target epmd@0.0.0.0.socket [Service] +# Note: You *may* wish to uncomment the following lines to apply systemd +# hardening effort to RabbitMQ, to prevent your system from being illegally +# modified by undiscovered vulnerabilities in RabbitMQ. +# ProtectSystem=full +# ProtectHome=true +# PrivateDevices=true +# ProtectHostname=true +# ProtectClock=true +# ProtectKernelTunables=true +# ProtectKernelModules=true +# ProtectKernelLogs=true +# ProtectControlGroups=true +# RestrictRealtime=true Type=notify User=rabbitmq Group=rabbitmq diff --git a/deps/rabbit/src/rabbit_channel.erl b/deps/rabbit/src/rabbit_channel.erl index 962ee864d9..b66c82381f 100644 --- a/deps/rabbit/src/rabbit_channel.erl +++ b/deps/rabbit/src/rabbit_channel.erl @@ -2170,7 +2170,7 @@ deliver_to_queues({Delivery = #delivery{message = MessageContainer, Qs = rabbit_amqqueue:lookup(AllNames), case rabbit_queue_type:deliver(Qs, Delivery, QueueStates0) of {ok, QueueStates, Actions} -> - rabbit_global_counters:messages_routed(amqp091, length(Qs)), + rabbit_global_counters:messages_routed(amqp091, erlang:min(1, length(Qs))), %% NB: the order here is important since basic.returns must be %% sent before confirms. ok = process_routing_mandatory(Mandatory, Qs, MessageContainer, State0), diff --git a/deps/rabbit/src/rabbit_fifo_client.erl b/deps/rabbit/src/rabbit_fifo_client.erl index 9b8920f89e..7f2cd55acc 100644 --- a/deps/rabbit/src/rabbit_fifo_client.erl +++ b/deps/rabbit/src/rabbit_fifo_client.erl @@ -42,7 +42,6 @@ -define(COMMAND_TIMEOUT, 30000). -type seq() :: non_neg_integer(). -%% last_applied is initialised to -1 -type maybe_seq() :: integer(). -type action() :: {send_credit_reply, Available :: non_neg_integer()} | {send_drained, CTagCredit :: @@ -300,7 +299,7 @@ settle(ConsumerTag, [_|_] = MsgIds, %% from {@link rabbit_fifo:delivery/0.} %% @param State the {@module} state %% @returns -%% `{ok | slow, State}' if the command was successfully sent. If the return +%% `{State, list()}' if the command was successfully sent. If the return %% tag is `slow' it means the limit is approaching and it is time to slow down %% the sending rate. %% @@ -310,10 +309,8 @@ return(ConsumerTag, [_|_] = MsgIds, #state{slow = false} = State0) -> Node = pick_server(State0), % TODO: make rabbit_fifo return support lists of message ids Cmd = rabbit_fifo:make_return(consumer_id(ConsumerTag), MsgIds), - case send_command(Node, undefined, Cmd, normal, State0) of - {_, S} -> - {S, []} - end; + {_Tag, State1} = send_command(Node, undefined, Cmd, normal, State0), + {State1, []}; return(ConsumerTag, [_|_] = MsgIds, #state{unsent_commands = Unsent0} = State0) -> ConsumerId = consumer_id(ConsumerTag), @@ -323,7 +320,8 @@ return(ConsumerTag, [_|_] = MsgIds, fun ({Settles, Returns, Discards}) -> {Settles, Returns ++ MsgIds, Discards} end, {[], MsgIds, []}, Unsent0), - {State0#state{unsent_commands = Unsent}, []}. + State1 = State0#state{unsent_commands = Unsent}, + {State1, []}. %% @doc Discards a checked out message. %% If the queue has a dead_letter_handler configured this will be called. @@ -732,10 +730,10 @@ maybe_auto_ack(false, {deliver, Tag, _Ack, Msgs} = Deliver, State0) -> {State, Actions} = settle(Tag, MsgIds, State0), {ok, State, [Deliver] ++ Actions}. - handle_delivery(Leader, {delivery, Tag, [{FstId, _} | _] = IdMsgs}, #state{cfg = #cfg{cluster_name = QName}, - consumer_deliveries = CDels0} = State0) -> + consumer_deliveries = CDels0} = State0) + when is_map_key(Tag, CDels0) -> QRef = qref(Leader), {LastId, _} = lists:last(IdMsgs), Consumer = #consumer{ack = Ack} = maps:get(Tag, CDels0), @@ -787,7 +785,17 @@ handle_delivery(Leader, {delivery, Tag, [{FstId, _} | _] = IdMsgs}, length(IdMsgs), C#consumer{last_msg_id = LastId}, CDels0)}) - end. + end; +handle_delivery(_Leader, {delivery, Tag, [_ | _] = IdMsgs}, + #state{consumer_deliveries = CDels0} = State0) + when not is_map_key(Tag, CDels0) -> + %% Note: + %% https://github.com/rabbitmq/rabbitmq-server/issues/3729 + %% If the consumer is no longer in the deliveries map, + %% we should return all messages. + MsgIntIds = [Id || {Id, _} <- IdMsgs], + {State1, Deliveries} = return(Tag, MsgIntIds, State0), + {ok, State1, Deliveries}. transform_msgs(QName, QRef, Msgs) -> lists:map( diff --git a/deps/rabbit/src/rabbit_stream_coordinator.erl b/deps/rabbit/src/rabbit_stream_coordinator.erl index 7e8069d866..5c23fbb51d 100644 --- a/deps/rabbit/src/rabbit_stream_coordinator.erl +++ b/deps/rabbit/src/rabbit_stream_coordinator.erl @@ -825,7 +825,10 @@ phase_update_mnesia(StreamId, Args, #{reference := QName, %% amqqueue record amqqueue:set_type_state( amqqueue:set_pid(Q, LeaderPid), Conf); - _ -> + Ts -> + S = maps:get(name, Ts, undefined), + rabbit_log:debug("~s: refusing mnesia update for stale stream id ~s, current ~s", + [?MODULE, StreamId, S]), %% if the stream id isn't a match this is a stale %% update from a previous stream incarnation for the %% same queue name and we ignore it @@ -837,10 +840,23 @@ phase_update_mnesia(StreamId, Args, #{reference := QName, rabbit_amqqueue:update(QName, Fun) end) of not_found -> + rabbit_log:debug("~s: resource for stream id ~s not found, " + "recovering from rabbit_durable_queue", + [?MODULE, StreamId]), %% This can happen during recovery + %% we need to re-initialise the queue record + %% if the stream id is a match [Q] = mnesia:dirty_read(rabbit_durable_queue, QName), - %% TODO: what is the possible return type here? - _ = rabbit_amqqueue:ensure_rabbit_queue_record_is_initialized(Fun(Q)), + case amqqueue:get_type_state(Q) of + #{name := S} when S == StreamId -> + rabbit_log:debug("~s: initializing queue record for stream id ~s", + [?MODULE, StreamId]), + _ = rabbit_amqqueue:ensure_rabbit_queue_record_is_initialized(Fun(Q)), + ok; + _ -> + ok + end, + send_self_command({mnesia_updated, StreamId, Args}); _ -> send_self_command({mnesia_updated, StreamId, Args}) diff --git a/deps/rabbit/test/queue_type_SUITE.erl b/deps/rabbit/test/queue_type_SUITE.erl index 64a562007b..8e4c2a39fa 100644 --- a/deps/rabbit/test/queue_type_SUITE.erl +++ b/deps/rabbit/test/queue_type_SUITE.erl @@ -163,7 +163,7 @@ smoke(Config) -> %% get and ack basic_ack(Ch, basic_get(Ch, QName)), %% global counters - publish_and_confirm(Ch, <<"inexistent_queue">>, <<"msg4">>), + publish_and_confirm(Ch, <<"non-existent_queue">>, <<"msg4">>), ConsumerTag3 = <<"ctag3">>, ok = subscribe(Ch, QName, ConsumerTag3), ProtocolCounters = maps:get([{protocol, amqp091}], get_global_counters(Config)), diff --git a/deps/rabbit/test/quorum_queue_SUITE.erl b/deps/rabbit/test/quorum_queue_SUITE.erl index cdbb59d12b..6b21c1e813 100644 --- a/deps/rabbit/test/quorum_queue_SUITE.erl +++ b/deps/rabbit/test/quorum_queue_SUITE.erl @@ -137,7 +137,8 @@ all_tests() -> delete_if_unused, queue_ttl, peek, - consumer_priorities + consumer_priorities, + cancel_consumer_gh_3729 ]. memory_tests() -> @@ -1981,7 +1982,7 @@ subscribe_redelivery_limit(Config) -> receive {#'basic.deliver'{redelivered = true}, #amqp_msg{}} -> throw(unexpected_redelivery) - after 2000 -> + after 5000 -> ok end. @@ -2027,7 +2028,7 @@ subscribe_redelivery_policy(Config) -> receive {#'basic.deliver'{redelivered = true}, #amqp_msg{}} -> throw(unexpected_redelivery) - after 2000 -> + after 5000 -> ok end, ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"delivery-limit">>). @@ -2766,6 +2767,54 @@ consumer_priorities(Config) -> ok. +cancel_consumer_gh_3729(Config) -> + %% Test the scenario where a message is published to a quorum queue + %% but the consumer has been cancelled + %% https://github.com/rabbitmq/rabbitmq-server/pull/3746 + QQ = ?config(queue_name, Config), + + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + + ExpectedDeclareRslt0 = #'queue.declare_ok'{queue = QQ, message_count = 0, consumer_count = 0}, + DeclareRslt0 = declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}]), + ?assertMatch(ExpectedDeclareRslt0, DeclareRslt0), + + ok = publish(Ch, QQ), + + ok = subscribe(Ch, QQ, false), + + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, _} -> + R = #'basic.reject'{delivery_tag = DeliveryTag, requeue = true}, + ok = amqp_channel:cast(Ch, R) + after 5000 -> + flush(100), + ct:fail("basic.deliver timeout") + end, + + ok = cancel(Ch), + + D = #'queue.declare'{queue = QQ, passive = true, arguments = [{<<"x-queue-type">>, longstr, <<"quorum">>}]}, + #'queue.declare_ok'{queue = QQ, message_count = 0, consumer_count = 1} = amqp_channel:call(Ch, D), + + receive + #'basic.cancel_ok'{consumer_tag = <<"ctag">>} -> ok + after 5000 -> + flush(100), + ct:fail("basic.cancel_ok timeout") + end, + + F = fun() -> + #'queue.declare_ok'{queue = QQ, + message_count = MC, + consumer_count = CC} = amqp_channel:call(Ch, D), + MC =:= 1 andalso CC =:= 0 + end, + wait_until(F), + + ok = rabbit_ct_client_helpers:close_channel(Ch). + %%---------------------------------------------------------------------------- declare(Ch, Q) -> @@ -2824,6 +2873,10 @@ qos(Ch, Prefetch, Global) -> amqp_channel:call(Ch, #'basic.qos'{global = Global, prefetch_count = Prefetch})). +cancel(Ch) -> + ?assertMatch(#'basic.cancel_ok'{consumer_tag = <<"ctag">>}, + amqp_channel:call(Ch, #'basic.cancel'{consumer_tag = <<"ctag">>})). + receive_basic_deliver(Redelivered) -> receive {#'basic.deliver'{redelivered = R}, _} when R == Redelivered -> @@ -2913,7 +2966,7 @@ validate_queue(Ch, Queue, ExpectedMsgs) -> #amqp_msg{payload = M}} -> amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag1, multiple = false}) - after 2000 -> + after 5000 -> flush(10), exit({validate_queue_timeout, M}) end diff --git a/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl b/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl index 859db2178f..85881f68cd 100644 --- a/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl +++ b/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl @@ -504,7 +504,7 @@ snapshots(_Config) -> collect({log_size, length(O)}, snapshots_prop(Config, O))) end) - end, [], 2500). + end, [], 1000). single_active(_Config) -> Size = 2000, @@ -1046,7 +1046,7 @@ handle_op({input_event, requeue}, #t{effects = Effs} = T) -> handle_op({input_event, Settlement}, #t{effects = Effs, down = Down} = T) -> case queue:out(Effs) of - {{value, {settle, MsgIds, CId}}, Q} -> + {{value, {settle, CId, MsgIds}}, Q} -> Cmd = case Settlement of settle -> rabbit_fifo:make_settle(CId, MsgIds); return -> rabbit_fifo:make_return(CId, MsgIds); @@ -1097,7 +1097,7 @@ do_apply(Cmd, #t{effects = Effs, end. enq_effs([], Q) -> Q; -enq_effs([{send_msg, P, {delivery, CTag, Msgs}, ra_event} | Rem], Q) -> +enq_effs([{send_msg, P, {delivery, CTag, Msgs}, _Opts} | Rem], Q) -> MsgIds = [I || {I, _} <- Msgs], %% always make settle commands by default %% they can be changed depending on the input event later diff --git a/deps/rabbitmq_auth_backend_oauth2/priv/schema/rabbitmq_auth_backend_oauth2.schema b/deps/rabbitmq_auth_backend_oauth2/priv/schema/rabbitmq_auth_backend_oauth2.schema index 010604647d..0feb73d9aa 100644 --- a/deps/rabbitmq_auth_backend_oauth2/priv/schema/rabbitmq_auth_backend_oauth2.schema +++ b/deps/rabbitmq_auth_backend_oauth2/priv/schema/rabbitmq_auth_backend_oauth2.schema @@ -49,7 +49,7 @@ %% A map of signing keys %% -%% {signing_keys, #{<<"id1">> => <<"value1">>,<<"id2">> => <<"value2">>}} +%% {signing_keys, #{<<"id1">> => {pem, <<"value1">>}, <<"id2">> => {pem, <<"value2">>}}} %% validator doesn't work {mapping, @@ -73,7 +73,7 @@ end, SigningKeys = lists:map(fun({Id, Path}) -> - {list_to_binary(lists:last(Id)), TryReadingFileFun(Path)} + {list_to_binary(lists:last(Id)), {pem, TryReadingFileFun(Path)}} end, Settings), maps:from_list(SigningKeys) end}. diff --git a/deps/rabbitmq_auth_backend_oauth2/test/config_schema_SUITE_data/rabbitmq_auth_backend_oauth2.snippets b/deps/rabbitmq_auth_backend_oauth2/test/config_schema_SUITE_data/rabbitmq_auth_backend_oauth2.snippets index a6d3d24a0f..2b7018fdd8 100644 --- a/deps/rabbitmq_auth_backend_oauth2/test/config_schema_SUITE_data/rabbitmq_auth_backend_oauth2.snippets +++ b/deps/rabbitmq_auth_backend_oauth2/test/config_schema_SUITE_data/rabbitmq_auth_backend_oauth2.snippets @@ -13,8 +13,8 @@ {default_key, <<"id1">>}, {signing_keys, #{ - <<"id1">> => <<"I'm not a certificate">>, - <<"id2">> => <<"I'm not a certificate">> + <<"id1">> => {pem, <<"I'm not a certificate">>}, + <<"id2">> => {pem, <<"I'm not a certificate">>} } } ] diff --git a/deps/rabbitmq_management_agent/priv/schema/rabbitmq_management_agent.schema b/deps/rabbitmq_management_agent/priv/schema/rabbitmq_management_agent.schema index fa8a76725a..3d66d9aaee 100644 --- a/deps/rabbitmq_management_agent/priv/schema/rabbitmq_management_agent.schema +++ b/deps/rabbitmq_management_agent/priv/schema/rabbitmq_management_agent.schema @@ -2,3 +2,4 @@ %% Also the management application will refuse to start if metrics collection is disabled {mapping, "management_agent.disable_metrics_collector", "rabbitmq_management_agent.disable_metrics_collector", [{datatype, {enum, [true, false]}}]}. +{mapping, "management_agent.filter_aggregated_queue_metrics_pattern", "rabbitmq_management_agent.filter_aggregated_queue_metrics_pattern", [{datatype, string}]}.
\ No newline at end of file diff --git a/deps/rabbitmq_management_agent/src/rabbit_mgmt_db_handler.erl b/deps/rabbitmq_management_agent/src/rabbit_mgmt_db_handler.erl index bcba92c6e4..1eaad7ff53 100644 --- a/deps/rabbitmq_management_agent/src/rabbit_mgmt_db_handler.erl +++ b/deps/rabbitmq_management_agent/src/rabbit_mgmt_db_handler.erl @@ -93,7 +93,15 @@ handle_info(_Info, State) -> {ok, State}. terminate(_Arg, _State) -> + ensure_statistics_disabled(), ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. + +ensure_statistics_disabled() -> + %% Reset the default values, see Makefile + _ = rabbit_log:info("Management plugin: to stop collect_statistics."), + application:set_env(rabbit, collect_statistics, none), + application:set_env(rabbit, collect_statistics_interval, 5000), + ok = rabbit:force_event_refresh(erlang:make_ref()). diff --git a/deps/rabbitmq_management_agent/src/rabbit_mgmt_metrics_collector.erl b/deps/rabbitmq_management_agent/src/rabbit_mgmt_metrics_collector.erl index 3df72d9233..9ab07fd808 100644 --- a/deps/rabbitmq_management_agent/src/rabbit_mgmt_metrics_collector.erl +++ b/deps/rabbitmq_management_agent/src/rabbit_mgmt_metrics_collector.erl @@ -26,7 +26,8 @@ -import(rabbit_mgmt_data, [lookup_element/3]). -record(state, {table, interval, policies, rates_mode, lookup_queue, - lookup_exchange, old_aggr_stats}). + lookup_exchange, old_aggr_stats, + filter_aggregated_queue_metrics_pattern}). %% Data is stored in ETS tables: %% * One ETS table per metric (queue_stats, channel_stats_deliver_stats...) @@ -59,7 +60,7 @@ reset_lookups(Table) -> gen_server:call(name(Table), reset_lookups, infinity). init([Table]) -> - {RatesMode, Policies} = load_config(), + {RatesMode, Policies, FilterPattern} = load_config(), Policy = retention_policy(Table), Interval = take_smaller(proplists:get_value(Policy, Policies, [])) * 1000, erlang:send_after(Interval, self(), collect_metrics), @@ -70,7 +71,8 @@ init([Table]) -> rates_mode = RatesMode, old_aggr_stats = #{}, lookup_queue = fun queue_exists/1, - lookup_exchange = fun exchange_exists/1}}. + lookup_exchange = fun exchange_exists/1, + filter_aggregated_queue_metrics_pattern = FilterPattern}}. handle_call(reset_lookups, _From, State) -> {reply, ok, State#state{lookup_queue = fun queue_exists/1, @@ -463,19 +465,19 @@ aggregate_entry({Name, Ready, Unack, Msgs, Red}, NextStats, Ops0, #state{table = queue_coarse_metrics, old_aggr_stats = Old, policies = {BPolicies, _, GPolicies}, - lookup_queue = QueueFun} = State) -> + lookup_queue = QueueFun, + filter_aggregated_queue_metrics_pattern = Pattern} = State) -> Stats = ?vhost_msg_stats(Ready, Unack, Msgs), Diff = get_difference(Name, Stats, State), - Ops1 = insert_entry_ops(vhost_msg_stats, vhost(Name), true, Diff, Ops0, - GPolicies), + Ops1 = maybe_insert_entry_ops(Name, Pattern, vhost_msg_stats, vhost(Name), + true, Diff, Ops0, GPolicies), Ops2 = case QueueFun(Name) of true -> QPS =?queue_process_stats(Red), O1 = insert_entry_ops(queue_process_stats, Name, false, QPS, Ops1, BPolicies), QMS = ?queue_msg_stats(Ready, Unack, Msgs), - insert_entry_ops(queue_msg_stats, Name, false, QMS, - O1, BPolicies); + insert_entry_ops(queue_msg_stats, Name, false, QMS, O1, BPolicies); _ -> Ops1 end, @@ -583,6 +585,17 @@ insert_entry_op(Table, Key, Entry, Ops) -> end, {insert_entry, Entry}, TableOps0), maps:put(Table, TableOps, Ops). +maybe_insert_entry_ops(Name, Pattern, Table, Id, Incr, Entry, Ops, Policies) -> + case needs_filtering_out(Name, Pattern) of + true -> Ops; + false -> insert_entry_ops(Table, Id, Incr, Entry, Ops, Policies) + end. + +needs_filtering_out(_, undefined) -> + false; +needs_filtering_out(#resource{name = Name}, Pattern) -> + match == re:run(Name, Pattern, [{capture, none}]). + insert_entry_ops(Table, Id, Incr, Entry, Ops, Policies) -> lists:foldl(fun({Size, Interval}, Acc) -> Key = {Id, Size, Interval, Incr}, @@ -688,7 +701,8 @@ index_table(node_node_coarse_stats, node) -> node_node_coarse_stats_node_index. load_config() -> RatesMode = rabbit_mgmt_agent_config:get_env(rates_mode), Policies = rabbit_mgmt_agent_config:get_env(sample_retention_policies, []), - {RatesMode, Policies}. + FilterPattern = rabbit_mgmt_agent_config:get_env(filter_aggregated_queue_metrics_pattern), + {RatesMode, Policies, FilterPattern}. ceil(X) when X < 0 -> trunc(X); diff --git a/deps/rabbitmq_prometheus/metrics-detailed.md b/deps/rabbitmq_prometheus/metrics-detailed.md index 262fca04bb..bec732f8db 100644 --- a/deps/rabbitmq_prometheus/metrics-detailed.md +++ b/deps/rabbitmq_prometheus/metrics-detailed.md @@ -236,3 +236,32 @@ Group `channel_queue_exchange_metrics`: | Metric | Description | |--------------------------------------------------|----------------------------------------------| | rabbitmq_detailed_queue_messages_published_total | Total number of messages published to queues | + +### Virtual hosts and exchange metrics + +These additional metrics can be useful when virtual hosts or exchanges are +created on a shared cluster in a self-service way. They are different +from the rest of the metrics: they are cluster-wide and not node-local. +These metrics **must not** be aggregated across cluster nodes. + +Group `vhost_status`: + +| Metric | Description | +|-------------------------------|----------------------------------| +| rabbitmq_cluster_vhost_status | Whether a given vhost is running | + +Group `exchange_names`: + +| Metric | Description | +|--------------------------------|----------------------------------------------------------------------------------------------------------------------------| +| rabbitmq_cluster_exchange_name | Enumerates exchanges without any additional info. This value is cluster-wide. A cheaper alternative to `exchange_bindings` | + +Group `exchange_bindings`: + +| Metric | Description | +|------------------------------------|-----------------------------------------------------------------| +| rabbitmq_cluster_exchange_bindings | Number of bindings for an exchange. This value is cluster-wide. | + + + + diff --git a/deps/rabbitmq_prometheus/priv/schema/rabbitmq_prometheus.schema b/deps/rabbitmq_prometheus/priv/schema/rabbitmq_prometheus.schema index a406f604fd..0ba11713f6 100644 --- a/deps/rabbitmq_prometheus/priv/schema/rabbitmq_prometheus.schema +++ b/deps/rabbitmq_prometheus/priv/schema/rabbitmq_prometheus.schema @@ -126,3 +126,5 @@ end}. [{datatype, integer}, {validators, ["non_negative_integer"]}]}. {mapping, "prometheus.ssl.max_keepalive", "rabbitmq_prometheus.ssl_config.cowboy_opts.max_keepalive", [{datatype, integer}, {validators, ["non_negative_integer"]}]}. + +{mapping, "prometheus.filter_aggregated_queue_metrics_pattern", "rabbitmq_prometheus.filter_aggregated_queue_metrics_pattern", [{datatype, string}]}.
\ No newline at end of file diff --git a/deps/rabbitmq_prometheus/src/collectors/prometheus_rabbitmq_core_metrics_collector.erl b/deps/rabbitmq_prometheus/src/collectors/prometheus_rabbitmq_core_metrics_collector.erl index 8b46770ef8..7c31b71b92 100644 --- a/deps/rabbitmq_prometheus/src/collectors/prometheus_rabbitmq_core_metrics_collector.erl +++ b/deps/rabbitmq_prometheus/src/collectors/prometheus_rabbitmq_core_metrics_collector.erl @@ -29,6 +29,7 @@ %% Used by `/metrics/detailed` endpoint -define(DETAILED_METRIC_NAME_PREFIX, <<"rabbitmq_detailed_">>). +-define(CLUSTER_METRIC_NAME_PREFIX, <<"rabbitmq_cluster_">>). %% ==The source of these metrics can be found in the rabbit_core_metrics module== %% The relevant files are: @@ -214,6 +215,19 @@ ]} ]). +%% Metrics that can be only requested through `/metrics/detailed` +-define(METRICS_CLUSTER,[ + {vhost_status, [ + {2, undefined, vhost_status, gauge, "Whether a given vhost is running"} + ]}, + {exchange_bindings, [ + {2, undefined, exchange_bindings, gauge, "Number of bindings for an exchange. This value is cluster-wide."} + ]}, + {exchange_names, [ + {2, undefined, exchange_name, gauge, "Enumerates exchanges without any additional info. This value is cluster-wide. A cheaper alternative to `exchange_bindings`"} + ]} +]). + -define(TOTALS, [ %% ordering differs from metrics above, refer to list comprehension {connection_created, connections, gauge, "Connections currently open"}, @@ -232,25 +246,26 @@ register() -> deregister_cleanup(_) -> ok. collect_mf('detailed', Callback) -> - collect(true, ?DETAILED_METRIC_NAME_PREFIX, vhosts_filter_from_pdict(), enabled_mfs_from_pdict(), Callback), + collect(true, ?DETAILED_METRIC_NAME_PREFIX, vhosts_filter_from_pdict(), queues_filter_from_pdict(), enabled_mfs_from_pdict(?METRICS_RAW), Callback), + collect(true, ?CLUSTER_METRIC_NAME_PREFIX, vhosts_filter_from_pdict(), queues_filter_from_pdict(), enabled_mfs_from_pdict(?METRICS_CLUSTER), Callback), %% identity is here to enable filtering on a cluster name (as already happens in existing dashboards) emit_identity_info(Callback), ok; collect_mf('per-object', Callback) -> - collect(true, ?METRIC_NAME_PREFIX, false, ?METRICS_RAW, Callback), + collect(true, ?METRIC_NAME_PREFIX, false, queues_filter_from_pdict(), ?METRICS_RAW, Callback), totals(Callback), emit_identity_info(Callback), ok; collect_mf(_Registry, Callback) -> PerObjectMetrics = application:get_env(rabbitmq_prometheus, return_per_object_metrics, false), - collect(PerObjectMetrics, ?METRIC_NAME_PREFIX, false, ?METRICS_RAW, Callback), + collect(PerObjectMetrics, ?METRIC_NAME_PREFIX, false, queues_filter_from_pdict(), ?METRICS_RAW, Callback), totals(Callback), emit_identity_info(Callback), ok. -collect(PerObjectMetrics, Prefix, VHostsFilter, IncludedMFs, Callback) -> +collect(PerObjectMetrics, Prefix, VHostsFilter, QueuesFilter, IncludedMFs, Callback) -> [begin - Data = get_data(Table, PerObjectMetrics, VHostsFilter), + Data = get_data(Table, PerObjectMetrics, VHostsFilter, QueuesFilter), mf(Callback, Prefix, Contents, Data) end || {Table, Contents} <- IncludedMFs, not mutually_exclusive_mf(PerObjectMetrics, Table, IncludedMFs)]. @@ -383,6 +398,14 @@ collect_metrics(_, {Type, Fun, Items}) -> labels(Item) -> label(element(1, Item)). +label(L) when is_binary(L) -> + L; +label(M) when is_map(M) -> + maps:fold(fun (K, V, Acc = <<>>) -> + <<Acc/binary, K/binary, "=\"", V/binary, "\"">>; + (K, V, Acc) -> + <<Acc/binary, ",", K/binary, "=\"", V/binary, "\"">> + end, <<>>, M); label(#resource{virtual_host = VHost, kind = exchange, name = Name}) -> <<"vhost=\"", VHost/binary, "\",exchange=\"", Name/binary, "\"">>; label(#resource{virtual_host = VHost, kind = queue, name = Name}) -> @@ -459,7 +482,7 @@ emit_gauge_metric_if_defined(Labels, Value) -> gauge_metric(Labels, Value) end. -get_data(connection_metrics = Table, false, _) -> +get_data(connection_metrics = Table, false, _, _) -> {Table, A1, A2, A3, A4} = ets:foldl(fun({_, Props}, {T, A1, A2, A3, A4}) -> {T, sum(proplists:get_value(recv_cnt, Props), A1), @@ -468,7 +491,7 @@ get_data(connection_metrics = Table, false, _) -> sum(proplists:get_value(channels, Props), A4)} end, empty(Table), Table), [{Table, [{recv_cnt, A1}, {send_cnt, A2}, {send_pend, A3}, {channels, A4}]}]; -get_data(channel_metrics = Table, false, _) -> +get_data(channel_metrics = Table, false, _, _) -> {Table, A1, A2, A3, A4, A5, A6, A7} = ets:foldl(fun({_, Props}, {T, A1, A2, A3, A4, A5, A6, A7}) -> {T, @@ -483,42 +506,42 @@ get_data(channel_metrics = Table, false, _) -> [{Table, [{consumer_count, A1}, {messages_unacknowledged, A2}, {messages_unconfirmed, A3}, {messages_uncommitted, A4}, {acks_uncommitted, A5}, {prefetch_count, A6}, {global_prefetch_count, A7}]}]; -get_data(queue_consumer_count = MF, false, VHostsFilter) -> +get_data(queue_consumer_count = MF, false, VHostsFilter, QueuesFilter) -> Table = queue_metrics, %% Real table name {_, A1} = ets:foldl(fun ({#resource{kind = queue, virtual_host = VHost}, _, _}, Acc) when is_map(VHostsFilter), map_get(VHost, VHostsFilter) == false -> Acc; + ({#resource{kind = queue, name = Name}, Props, _}, {T, A1} = Acc) + when is_list(QueuesFilter) -> + case re:run(Name, QueuesFilter, [{capture, none}]) of + match -> + Acc; + nomatch -> + {T, + sum(proplists:get_value(consumers, Props), A1) + } + end; ({_, Props, _}, {T, A1}) -> {T, sum(proplists:get_value(consumers, Props), A1) } end, empty(MF), Table), [{Table, [{consumers, A1}]}]; -get_data(queue_metrics = Table, false, VHostsFilter) -> +get_data(queue_metrics = Table, false, VHostsFilter, QueuesFilter) -> {Table, A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16} = ets:foldl(fun ({#resource{kind = queue, virtual_host = VHost}, _, _}, Acc) when is_map(VHostsFilter), map_get(VHost, VHostsFilter) == false -> Acc; - ({_, Props, _}, {T, A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, - A11, A12, A13, A14, A15, A16}) -> - {T, - sum(proplists:get_value(consumers, Props), A1), - sum(proplists:get_value(consumer_utilisation, Props), A2), - sum(proplists:get_value(memory, Props), A3), - sum(proplists:get_value(messages_ram, Props), A4), - sum(proplists:get_value(message_bytes_ram, Props), A5), - sum(proplists:get_value(messages_ready_ram, Props), A6), - sum(proplists:get_value(messages_unacknowledged_ram, Props), A7), - sum(proplists:get_value(messages_persistent, Props), A8), - sum(proplists:get_value(message_bytes_persistent, Props), A9), - sum(proplists:get_value(message_bytes, Props), A10), - sum(proplists:get_value(message_bytes_ready, Props), A11), - sum(proplists:get_value(message_bytes_unacknowledged, Props), A12), - sum(proplists:get_value(messages_paged_out, Props), A13), - sum(proplists:get_value(message_bytes_paged_out, Props), A14), - sum(proplists:get_value(disk_reads, Props), A15), - sum(proplists:get_value(disk_writes, Props), A16) - } + ({#resource{kind = queue, name = Name}, Props, _}, Acc) + when is_list(QueuesFilter) -> + case re:run(Name, QueuesFilter, [{capture, none}]) of + match -> + Acc; + nomatch -> + sum_queue_metrics(Props, Acc) + end; + ({_, Props, _}, Acc) -> + sum_queue_metrics(Props, Acc) end, empty(Table), Table), [{Table, [{consumers, A1}, {consumer_utilisation, A2}, {memory, A3}, {messages_ram, A4}, {message_bytes_ram, A5}, {messages_ready_ram, A6}, @@ -527,7 +550,7 @@ get_data(queue_metrics = Table, false, VHostsFilter) -> {message_bytes_ready, A11}, {message_bytes_unacknowledged, A12}, {messages_paged_out, A13}, {message_bytes_paged_out, A14}, {disk_reads, A15}, {disk_writes, A16}]}]; -get_data(Table, false, VHostsFilter) when Table == channel_exchange_metrics; +get_data(Table, false, VHostsFilter, QueuesFilter) when Table == channel_exchange_metrics; Table == queue_coarse_metrics; Table == channel_queue_metrics; Table == connection_coarse_metrics; @@ -538,6 +561,14 @@ get_data(Table, false, VHostsFilter) when Table == channel_exchange_metrics; %% For queue_coarse_metrics ({#resource{kind = queue, virtual_host = VHost}, _, _, _, _}, Acc) when is_map(VHostsFilter), map_get(VHost, VHostsFilter) == false -> Acc; + ({#resource{kind = queue, name = Name}, V1, V2, V3, V4}, {T, A1, A2, A3, A4} = Acc) + when is_list(QueuesFilter) -> + case re:run(Name, QueuesFilter, [{capture, none}]) of + match -> + Acc; + nomatch -> + {T, V1 + A1, V2 + A2, V3 + A3, V4 + A4} + end; ({_, V1}, {T, A1}) -> {T, V1 + A1}; ({_, V1, _}, {T, A1}) -> @@ -564,14 +595,14 @@ get_data(Table, false, VHostsFilter) when Table == channel_exchange_metrics; _ -> [Result] end; -get_data(queue_coarse_metrics = Table, true, VHostsFilter) when is_map(VHostsFilter) -> +get_data(queue_coarse_metrics = Table, true, VHostsFilter, _) when is_map(VHostsFilter) -> ets:foldl(fun ({#resource{kind = queue, virtual_host = VHost}, _, _, _, _} = Row, Acc) when map_get(VHost, VHostsFilter) -> [Row|Acc]; (_, Acc) -> Acc end, [], Table); -get_data(MF, true, VHostsFilter) when is_map(VHostsFilter), MF == queue_metrics orelse MF == queue_consumer_count -> +get_data(MF, true, VHostsFilter, _) when is_map(VHostsFilter), MF == queue_metrics orelse MF == queue_consumer_count -> Table = queue_metrics, ets:foldl(fun ({#resource{kind = queue, virtual_host = VHost}, _, _} = Row, Acc) when map_get(VHost, VHostsFilter) -> @@ -579,11 +610,73 @@ get_data(MF, true, VHostsFilter) when is_map(VHostsFilter), MF == queue_metrics (_, Acc) -> Acc end, [], Table); -get_data(queue_consumer_count, true, _) -> +get_data(queue_consumer_count, true, _, _) -> ets:tab2list(queue_metrics); -get_data(Table, _, _) -> +get_data(vhost_status, _, _, _) -> + [ { #{<<"vhost">> => VHost}, + case rabbit_vhost_sup_sup:is_vhost_alive(VHost) of + true -> 1; + false -> 0 + end} + || VHost <- rabbit_vhost:list() ]; +get_data(exchange_bindings, _, _, _) -> + Exchanges = lists:foldl(fun + (#exchange{internal = true}, Acc) -> + Acc; + (#exchange{name = #resource{name = <<>>}}, Acc) -> + Acc; + (#exchange{name = EName, type = EType}, Acc) -> + maps:put(EName, #{type => atom_to_binary(EType), binding_count => 0}, Acc) + end, #{}, rabbit_exchange:list()), + WithCount = ets:foldl( + fun (#route{binding = #binding{source = EName}}, Acc) -> + case maps:is_key(EName, Acc) of + false -> Acc; + true -> + maps:update_with(EName, fun (R = #{binding_count := Cnt}) -> + R#{binding_count => Cnt + 1} + end, Acc) + end + end, Exchanges, rabbit_route), + maps:fold(fun(#resource{virtual_host = VHost, name = Name}, #{type := Type, binding_count := Bindings}, Acc) -> + [{<<"vhost=\"", VHost/binary, "\",exchange=\"", Name/binary, "\",type=\"", Type/binary, "\"">>, + Bindings}|Acc] + end, [], WithCount); +get_data(exchange_names, _, _, _) -> + lists:foldl(fun + (#exchange{internal = true}, Acc) -> + Acc; + (#exchange{name = #resource{name = <<>>}}, Acc) -> + Acc; + (#exchange{name = #resource{virtual_host = VHost, name = Name}, type = EType}, Acc) -> + Label = <<"vhost=\"", VHost/binary, "\",exchange=\"", Name/binary, "\",type=\"", (atom_to_binary(EType))/binary, "\"">>, + [{Label, 1}|Acc] + end, [], rabbit_exchange:list()); +get_data(Table, _, _, _) -> ets:tab2list(Table). + +sum_queue_metrics(Props, {T, A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, + A12, A13, A14, A15, A16}) -> + {T, + sum(proplists:get_value(consumers, Props), A1), + sum(proplists:get_value(consumer_utilisation, Props), A2), + sum(proplists:get_value(memory, Props), A3), + sum(proplists:get_value(messages_ram, Props), A4), + sum(proplists:get_value(message_bytes_ram, Props), A5), + sum(proplists:get_value(messages_ready_ram, Props), A6), + sum(proplists:get_value(messages_unacknowledged_ram, Props), A7), + sum(proplists:get_value(messages_persistent, Props), A8), + sum(proplists:get_value(message_bytes_persistent, Props), A9), + sum(proplists:get_value(message_bytes, Props), A10), + sum(proplists:get_value(message_bytes_ready, Props), A11), + sum(proplists:get_value(message_bytes_unacknowledged, Props), A12), + sum(proplists:get_value(messages_paged_out, Props), A13), + sum(proplists:get_value(message_bytes_paged_out, Props), A14), + sum(proplists:get_value(disk_reads, Props), A15), + sum(proplists:get_value(disk_writes, Props), A16) + }. + division(0, 0) -> 0; division(A, B) -> @@ -612,13 +705,13 @@ sum('', B) -> sum(A, B) -> A + B. -enabled_mfs_from_pdict() -> +enabled_mfs_from_pdict(AllMFs) -> case get(prometheus_mf_filter) of undefined -> []; MFNames -> MFNameSet = sets:from_list(MFNames), - [ MF || MF = {Table, _} <- ?METRICS_RAW, sets:is_element(Table, MFNameSet) ] + [ MF || MF = {Table, _} <- AllMFs, sets:is_element(Table, MFNameSet) ] end. vhosts_filter_from_pdict() -> @@ -631,3 +724,11 @@ vhosts_filter_from_pdict() -> Enabled = maps:from_list([ {VHost, true} || VHost <- L ]), maps:merge(All, Enabled) end. + +queues_filter_from_pdict() -> + case get(prometheus_queue_filter) of + undefined -> + false; + Pattern -> + Pattern + end. diff --git a/deps/rabbitmq_prometheus/src/rabbit_prometheus_handler.erl b/deps/rabbitmq_prometheus/src/rabbit_prometheus_handler.erl index 954e2f878d..f7f4f11720 100644 --- a/deps/rabbitmq_prometheus/src/rabbit_prometheus_handler.erl +++ b/deps/rabbitmq_prometheus/src/rabbit_prometheus_handler.erl @@ -163,6 +163,12 @@ put_filtering_options_into_process_dictionary(Request) -> put(prometheus_mf_filter, Fs); _ -> ok end, + case application:get_env(rabbitmq_prometheus, filter_aggregated_queue_metrics_pattern, undefined) of + undefined -> ok; + Pattern -> + {ok, CompiledPattern} = re:compile(Pattern), + put(prometheus_queue_filter, CompiledPattern) + end, ok. parse_vhosts(N) when is_binary(N) -> diff --git a/deps/rabbitmq_prometheus/test/rabbit_prometheus_http_SUITE.erl b/deps/rabbitmq_prometheus/test/rabbit_prometheus_http_SUITE.erl index 10c573988c..c8be5fae28 100644 --- a/deps/rabbitmq_prometheus/test/rabbit_prometheus_http_SUITE.erl +++ b/deps/rabbitmq_prometheus/test/rabbit_prometheus_http_SUITE.erl @@ -54,7 +54,10 @@ groups() -> queue_consumer_count_all_vhosts_per_object_test, queue_coarse_metrics_per_object_test, queue_metrics_per_object_test, - queue_consumer_count_and_queue_metrics_mutually_exclusive_test + queue_consumer_count_and_queue_metrics_mutually_exclusive_test, + vhost_status_metric, + exchange_bindings_metric, + exchange_names_metric ]} ]. @@ -120,6 +123,14 @@ init_per_group(detailed_metrics, Config0) -> amqp_channel:cast(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = <<"msg">>}) + end, lists:seq(1, MsgNum) ), + ExDirect = <<QName/binary, "-direct-exchange">>, + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = ExDirect}), + ExTopic = <<QName/binary, "-topic-exchange">>, + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = ExTopic, type = <<"topic">>}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = QName, exchange = ExDirect, routing_key = QName}), + lists:foreach( fun (Idx) -> + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = QName, exchange = ExTopic, routing_key = integer_to_binary(Idx)}) end, lists:seq(1, MsgNum) ) end)() || {VHost, Ch, MsgNum} <- [{<<"/">>, DefaultCh, 3}, {<<"vhost-1">>, VHost1Ch, 7}, {<<"vhost-2">>, VHost2Ch, 11}], @@ -488,6 +499,51 @@ detailed_metrics_no_families_enabled_by_default(Config) -> ?assertEqual(#{}, parse_response(Body)), ok. +vhost_status_metric(Config) -> + {_, Body1} = http_get_with_pal(Config, "/metrics/detailed?family=vhost_status", [], 200), + Expected = #{rabbitmq_cluster_vhost_status => + #{#{vhost => "vhost-1"} => [1], + #{vhost => "vhost-2"} => [1], + #{vhost => "/"} => [1]}}, + ?assertEqual(Expected, parse_response(Body1)), + ok. + +exchange_bindings_metric(Config) -> + {_, Body1} = http_get_with_pal(Config, "/metrics/detailed?family=exchange_bindings", [], 200), + + Bindings = map_get(rabbitmq_cluster_exchange_bindings, parse_response(Body1)), + ?assertEqual([11], map_get(#{vhost=>"vhost-2",exchange=>"vhost-2-queue-with-messages-topic-exchange",type=>"topic"}, Bindings)), + ?assertEqual([1], map_get(#{vhost=>"vhost-2",exchange=>"vhost-2-queue-with-messages-direct-exchange",type=>"direct"}, Bindings)), + ok. + +exchange_names_metric(Config) -> + {_, Body1} = http_get_with_pal(Config, "/metrics/detailed?family=exchange_names", [], 200), + + Names = maps:filter( + fun + (#{exchange := [$a, $m, $q|_]}, _) -> + false; + (_, _) -> + true + end, + map_get(rabbitmq_cluster_exchange_name, parse_response(Body1))), + + ?assertEqual(#{ #{vhost=>"vhost-2",exchange=>"vhost-2-queue-with-messages-topic-exchange",type=>"topic"} => [1], + #{vhost=>"vhost-2",exchange=>"vhost-2-queue-with-messages-direct-exchange",type=>"direct"} => [1], + #{vhost=>"vhost-1",exchange=>"vhost-1-queue-with-messages-topic-exchange",type=>"topic"} => [1], + #{vhost=>"vhost-1",exchange=>"vhost-1-queue-with-messages-direct-exchange",type=>"direct"} => [1], + #{vhost=>"/",exchange=>"default-queue-with-messages-topic-exchange",type=>"topic"} => [1], + #{vhost=>"/",exchange=>"default-queue-with-messages-direct-exchange",type=>"direct"} => [1], + #{vhost=>"vhost-2",exchange=>"vhost-2-queue-with-consumer-topic-exchange",type=>"topic"} => [1], + #{vhost=>"vhost-2",exchange=>"vhost-2-queue-with-consumer-direct-exchange",type=>"direct"} => [1], + #{vhost=>"vhost-1",exchange=>"vhost-1-queue-with-consumer-topic-exchange",type=>"topic"} => [1], + #{vhost=>"vhost-1",exchange=>"vhost-1-queue-with-consumer-direct-exchange",type=>"direct"} => [1], + #{vhost=>"/",exchange=>"default-queue-with-consumer-topic-exchange",type=>"topic"} => [1], + #{vhost=>"/",exchange=>"default-queue-with-consumer-direct-exchange",type=>"direct"} => [1] + }, Names), + ok. + + http_get(Config, ReqHeaders, CodeExp) -> Path = proplists:get_value(prometheus_path, Config, "/metrics"), http_get(Config, Path, ReqHeaders, CodeExp). diff --git a/deps/rabbitmq_stream/docs/PROTOCOL.adoc b/deps/rabbitmq_stream/docs/PROTOCOL.adoc index 34786dad30..60a8e48d3e 100644 --- a/deps/rabbitmq_stream/docs/PROTOCOL.adoc +++ b/deps/rabbitmq_stream/docs/PROTOCOL.adoc @@ -322,6 +322,9 @@ Subscribe => Key Version CorrelationId SubscriptionId Stream OffsetSpecification Value => string ``` +NB: Timestamp is https://www.erlang.org/doc/apps/erts/time_correction.html#Erlang_System_Time[Erlang system time], +milliseconds from epoch + === Deliver ``` @@ -334,7 +337,7 @@ Deliver => Key Version SubscriptionId OsirisChunk ChunkType => int8 // 0: user, 1: tracking delta, 2: tracking snapshot NumEntries => uint16 NumRecords => uint32 - Timestamp => int64 // in milliseconds, since epoch + Timestamp => int64 // erlang system time in milliseconds, since epoch Epoch => uint64 ChunkFirstOffset => uint64 ChunkCrc => int32 @@ -346,7 +349,7 @@ Deliver => Key Version SubscriptionId OsirisChunk Data => bytes ``` -NB: See the https://github.com/rabbitmq/osiris/blob/348db0528986d6025b823bcf1ae0570aa63f5e25/src/osiris_log.erl#L49-L81[Osiris project] +NB: See the https://github.com/rabbitmq/osiris/blob/f32df7563a036b1687c0208a3cb5f9e8f5cee937/src/osiris_log.erl#L101[Osiris project] for details on the structure of messages. === Credit diff --git a/deps/rabbitmq_web_dispatch/BUILD.bazel b/deps/rabbitmq_web_dispatch/BUILD.bazel index d8395dfaab..096d91d0cd 100644 --- a/deps/rabbitmq_web_dispatch/BUILD.bazel +++ b/deps/rabbitmq_web_dispatch/BUILD.bazel @@ -1,4 +1,3 @@ -load("@bazel-erlang//:bazel_erlang_lib.bzl", "erlang_lib", "test_erlang_lib") load("@bazel-erlang//:xref.bzl", "xref") load("@bazel-erlang//:dialyze.bzl", "dialyze", "plt") load("//:rabbitmq_home.bzl", "rabbitmq_home") @@ -9,6 +8,7 @@ load( "RABBITMQ_DIALYZER_OPTS", "assert_suites", "rabbitmq_integration_suite", + "rabbitmq_lib", "rabbitmq_suite", ) @@ -28,16 +28,7 @@ DEPS = [ "@cowboy//:bazel_erlang_lib", ] -erlang_lib( - app_description = APP_DESCRIPTION, - app_module = APP_MODULE, - app_name = APP_NAME, - app_version = APP_VERSION, - extra_apps = EXTRA_APPS, - deps = DEPS, -) - -test_erlang_lib( +rabbitmq_lib( app_description = APP_DESCRIPTION, app_module = APP_MODULE, app_name = APP_NAME, diff --git a/dist.bzl b/dist.bzl new file mode 100644 index 0000000000..98eed9d1a9 --- /dev/null +++ b/dist.bzl @@ -0,0 +1,199 @@ +load("@bazel-erlang//:erlang_home.bzl", "ErlangHomeProvider") +load("@bazel-erlang//:bazel_erlang_lib.bzl", "ErlangLibInfo", "flat_deps", "path_join") +load("@bazel-erlang//:ct.bzl", "additional_file_dest_relative_path") +load( + ":rabbitmq_home.bzl", + "RABBITMQ_HOME_ATTRS", + "RabbitmqHomeInfo", + "flatten", + "link_escript", + "unique_versions", +) + +def _collect_licenses_impl(ctx): + srcs = ctx.files.srcs + flatten([ + d[ErlangLibInfo].license_files + for d in flat_deps(ctx.attr.deps) + ]) + + outs = {} + for src in srcs: + name = src.basename + if name not in outs: + dest = ctx.actions.declare_file(name) + ctx.actions.run( + inputs = [src], + outputs = [dest], + executable = "cp", + arguments = [ + src.path, + dest.path, + ], + ) + outs[name] = dest + + return [ + DefaultInfo( + files = depset(sorted(outs.values())), + ), + ] + +collect_licenses = rule( + implementation = _collect_licenses_impl, + attrs = { + "srcs": attr.label_list(allow_files = True), + "deps": attr.label_list(providers = [ErlangLibInfo]), + }, +) + +def _copy_script(ctx, script): + dest = ctx.actions.declare_file(path_join(ctx.label.name, "sbin", script.basename)) + ctx.actions.expand_template( + template = script, + output = dest, + substitutions = { + "SYS_PREFIX=": "SYS_PREFIX=${RABBITMQ_HOME}", + }, + ) + return dest + +def _extract_version(p): + return "erl -eval '{ok, [{application, _, AppInfo}]} = file:consult(\"" + p + "\"), Version = proplists:get_value(vsn, AppInfo), io:fwrite(Version), halt().' -noshell" + +def _app_file(plugin_lib_info): + for f in plugin_lib_info.beam: + if f.basename.endswith(".app"): + return f + fail(".app file not found in {}".format(plugin_lib_info)) + +def _plugins_dir(ctx, plugins): + plugins_dir = ctx.actions.declare_directory(path_join(ctx.label.name, "plugins")) + + erlang_home = ctx.attr._erlang_home[ErlangHomeProvider].path + + inputs = [] + + commands = ["set -euo pipefail", ""] + + for plugin in plugins: + lib_info = plugin[ErlangLibInfo] + app_file = _app_file(lib_info) + extract_version = _extract_version(app_file.path) + commands.append("PLUGIN_VERSION=$({erlang_home}/bin/{extract_version})".format(erlang_home = erlang_home, extract_version = extract_version)) + + commands.append( + "echo \"Assembling {lib_name}-$PLUGIN_VERSION...\"".format( + lib_name = lib_info.lib_name, + ), + ) + + commands.append( + "mkdir -p {plugins_dir}/{lib_name}-$PLUGIN_VERSION/include".format( + plugins_dir = plugins_dir.path, + lib_name = lib_info.lib_name, + ), + ) + for f in lib_info.include: + commands.append( + "cp {src} {plugins_dir}/{lib_name}-$PLUGIN_VERSION/include/{dest}".format( + src = f.path, + plugins_dir = plugins_dir.path, + lib_name = lib_info.lib_name, + dest = f.basename, + ), + ) + inputs.extend(lib_info.include) + + commands.append( + "mkdir -p {plugins_dir}/{lib_name}-$PLUGIN_VERSION/ebin".format( + plugins_dir = plugins_dir.path, + lib_name = lib_info.lib_name, + ), + ) + for f in lib_info.beam: + if f.is_directory: + if f.basename != "ebin": + fail("{} contains a directory in 'beam' that is not an ebin dir".format(lib_info.lib_name)) + commands.append( + "cp -R {src} {plugins_dir}/{lib_name}-$PLUGIN_VERSION".format( + src = f.path, + plugins_dir = plugins_dir.path, + lib_name = lib_info.lib_name, + ), + ) + else: + commands.append( + "cp {src} {plugins_dir}/{lib_name}-$PLUGIN_VERSION/ebin/{dest}".format( + src = f.path, + plugins_dir = plugins_dir.path, + lib_name = lib_info.lib_name, + dest = f.basename, + ), + ) + inputs.extend(lib_info.beam) + + for f in lib_info.priv: + p = additional_file_dest_relative_path(plugin.label, f) + commands.append( + "mkdir -p $(dirname {plugins_dir}/{lib_name}-$PLUGIN_VERSION/{dest}) && cp {src} {plugins_dir}/{lib_name}-$PLUGIN_VERSION/{dest}".format( + src = f.path, + plugins_dir = plugins_dir.path, + lib_name = lib_info.lib_name, + dest = p, + ), + ) + inputs.extend(lib_info.priv) + + commands.append("") + + ctx.actions.run_shell( + inputs = inputs, + outputs = [plugins_dir], + command = "\n".join(commands), + ) + + return plugins_dir + +def _versioned_rabbitmq_home_impl(ctx): + plugins = flat_deps(ctx.attr.plugins) + + erlang_versions = unique_versions(plugins) + if len(erlang_versions) > 1: + fail("plugins do not have a unified erlang version", erlang_versions) + + scripts = [_copy_script(ctx, script) for script in ctx.files._scripts] + + rabbitmq_ctl_copies = [ + "rabbitmq-diagnostics", + "rabbitmq-plugins", + "rabbitmq-queues", + "rabbitmq-streams", + "rabbitmq-upgrade", + "rabbitmqctl", + ] + escripts = [link_escript(ctx, escript) for escript in rabbitmq_ctl_copies] + + plugins_dir = _plugins_dir(ctx, plugins) + + rabbitmqctl = None + for script in scripts: + if script.basename == "rabbitmqctl": + rabbitmqctl = script + if rabbitmqctl == None: + fail("could not find rabbitmqct among", scripts) + + return [ + RabbitmqHomeInfo( + rabbitmqctl = rabbitmqctl, + ), + DefaultInfo( + files = depset(scripts + escripts + [plugins_dir]), + ), + ] + +versioned_rabbitmq_home = rule( + implementation = _versioned_rabbitmq_home_impl, + attrs = dict(RABBITMQ_HOME_ATTRS.items() + { + "_erlang_home": attr.label(default = "@bazel-erlang//:erlang_home"), + }.items()), +) diff --git a/rabbitmq.bzl b/rabbitmq.bzl index 4a5cdb489b..8e73b5922e 100644 --- a/rabbitmq.bzl +++ b/rabbitmq.bzl @@ -22,6 +22,42 @@ RABBITMQ_DIALYZER_OPTS = [ APP_VERSION = "3.10.0" +ALL_PLUGINS = [ + "//deps/rabbit:bazel_erlang_lib", + "//deps/rabbitmq_amqp1_0:bazel_erlang_lib", + "//deps/rabbitmq_auth_backend_cache:bazel_erlang_lib", + "//deps/rabbitmq_auth_backend_http:bazel_erlang_lib", + "//deps/rabbitmq_auth_backend_ldap:bazel_erlang_lib", + "//deps/rabbitmq_auth_backend_oauth2:bazel_erlang_lib", + "//deps/rabbitmq_auth_mechanism_ssl:bazel_erlang_lib", + "//deps/rabbitmq_consistent_hash_exchange:bazel_erlang_lib", + "//deps/rabbitmq_event_exchange:bazel_erlang_lib", + "//deps/rabbitmq_federation:bazel_erlang_lib", + "//deps/rabbitmq_federation_management:bazel_erlang_lib", + "//deps/rabbitmq_jms_topic_exchange:bazel_erlang_lib", + "//deps/rabbitmq_management:bazel_erlang_lib", + "//deps/rabbitmq_mqtt:bazel_erlang_lib", + "//deps/rabbitmq_peer_discovery_aws:bazel_erlang_lib", + "//deps/rabbitmq_peer_discovery_consul:bazel_erlang_lib", + "//deps/rabbitmq_peer_discovery_etcd:bazel_erlang_lib", + "//deps/rabbitmq_peer_discovery_k8s:bazel_erlang_lib", + "//deps/rabbitmq_prometheus:bazel_erlang_lib", + "//deps/rabbitmq_random_exchange:bazel_erlang_lib", + "//deps/rabbitmq_recent_history_exchange:bazel_erlang_lib", + "//deps/rabbitmq_sharding:bazel_erlang_lib", + "//deps/rabbitmq_shovel:bazel_erlang_lib", + "//deps/rabbitmq_shovel_management:bazel_erlang_lib", + "//deps/rabbitmq_stomp:bazel_erlang_lib", + "//deps/rabbitmq_stream:bazel_erlang_lib", + "//deps/rabbitmq_stream_management:bazel_erlang_lib", + "//deps/rabbitmq_top:bazel_erlang_lib", + "//deps/rabbitmq_tracing:bazel_erlang_lib", + "//deps/rabbitmq_trust_store:bazel_erlang_lib", + "//deps/rabbitmq_web_dispatch:bazel_erlang_lib", + "//deps/rabbitmq_web_mqtt:bazel_erlang_lib", + "//deps/rabbitmq_web_stomp:bazel_erlang_lib", +] + LABELS_WITH_TEST_VERSIONS = [ "//deps/amqp10_common:bazel_erlang_lib", "//deps/rabbit_common:bazel_erlang_lib", @@ -29,6 +65,9 @@ LABELS_WITH_TEST_VERSIONS = [ "//deps/rabbit/apps/rabbitmq_prelaunch:bazel_erlang_lib", ] +def all_plugins(rabbitmq_workspace = "@rabbitmq-server"): + return [rabbitmq_workspace + p for p in ALL_PLUGINS] + def with_test_versions(deps): r = [] for d in deps: diff --git a/rabbitmq_home.bzl b/rabbitmq_home.bzl index 6342d75f40..eca1ff1fb9 100644 --- a/rabbitmq_home.bzl +++ b/rabbitmq_home.bzl @@ -1,4 +1,5 @@ load("@bazel-erlang//:bazel_erlang_lib.bzl", "ErlangLibInfo", "flat_deps", "path_join") +load("@bazel-erlang//:ct.bzl", "additional_file_dest_relative_path") RabbitmqHomeInfo = provider( doc = "An assembled RABBITMQ_HOME dir", @@ -19,7 +20,7 @@ def _copy_script(ctx, script): ) return dest -def _link_escript(ctx, escript): +def link_escript(ctx, escript): e = ctx.attr._rabbitmqctl_escript.files_to_run.executable s = ctx.actions.declare_file(path_join(ctx.label.name, "escript", escript)) ctx.actions.symlink( @@ -28,19 +29,6 @@ def _link_escript(ctx, escript): ) return s -def _priv_file_dest_relative_path(plugin_label, f): - if plugin_label.workspace_root != "": - if plugin_label.package != "": - rel_base = path_join(plugin_label.workspace_root, plugin_label.package) - else: - rel_base = plugin_label.workspace_root - else: - rel_base = plugin_label.package - if rel_base != "": - return f.path.replace(rel_base + "/", "") - else: - return f.path - def _plugins_dir_links(ctx, plugin): lib_info = plugin[ErlangLibInfo] plugin_path = path_join( @@ -72,7 +60,7 @@ def _plugins_dir_links(ctx, plugin): links.append(o) for f in lib_info.priv: - p = _priv_file_dest_relative_path(plugin.label, f) + p = additional_file_dest_relative_path(plugin.label, f) o = ctx.actions.declare_file(path_join(plugin_path, p)) ctx.actions.symlink( output = o, @@ -82,7 +70,7 @@ def _plugins_dir_links(ctx, plugin): return links -def _unique_versions(plugins): +def unique_versions(plugins): erlang_versions = [] for plugin in plugins: erlang_version = plugin[ErlangLibInfo].erlang_version @@ -90,13 +78,13 @@ def _unique_versions(plugins): erlang_versions.append(erlang_version) return erlang_versions -def _flatten(list_of_lists): +def flatten(list_of_lists): return [item for sublist in list_of_lists for item in sublist] def _impl(ctx): plugins = flat_deps(ctx.attr.plugins) - erlang_versions = _unique_versions(plugins) + erlang_versions = unique_versions(plugins) if len(erlang_versions) > 1: fail("plugins do not have a unified erlang version", erlang_versions) @@ -110,9 +98,9 @@ def _impl(ctx): "rabbitmq-upgrade", "rabbitmqctl", ] - escripts = [_link_escript(ctx, escript) for escript in rabbitmq_ctl_copies] + escripts = [link_escript(ctx, escript) for escript in rabbitmq_ctl_copies] - plugins = _flatten([_plugins_dir_links(ctx, plugin) for plugin in plugins]) + plugins = flatten([_plugins_dir_links(ctx, plugin) for plugin in plugins]) rabbitmqctl = None for script in scripts: @@ -130,24 +118,26 @@ def _impl(ctx): ), ] +RABBITMQ_HOME_ATTRS = { + "_scripts": attr.label_list( + default = [ + "//deps/rabbit:scripts/rabbitmq-defaults", + "//deps/rabbit:scripts/rabbitmq-diagnostics", + "//deps/rabbit:scripts/rabbitmq-env", + "//deps/rabbit:scripts/rabbitmq-plugins", + "//deps/rabbit:scripts/rabbitmq-queues", + "//deps/rabbit:scripts/rabbitmq-server", + "//deps/rabbit:scripts/rabbitmqctl", + ], + allow_files = True, + ), + "_rabbitmqctl_escript": attr.label(default = "//deps/rabbitmq_cli:rabbitmqctl"), + "plugins": attr.label_list(providers = [ErlangLibInfo]), +} + rabbitmq_home = rule( implementation = _impl, - attrs = { - "_scripts": attr.label_list( - default = [ - "//deps/rabbit:scripts/rabbitmq-defaults", - "//deps/rabbit:scripts/rabbitmq-diagnostics", - "//deps/rabbit:scripts/rabbitmq-env", - "//deps/rabbit:scripts/rabbitmq-plugins", - "//deps/rabbit:scripts/rabbitmq-queues", - "//deps/rabbit:scripts/rabbitmq-server", - "//deps/rabbit:scripts/rabbitmqctl", - ], - allow_files = True, - ), - "_rabbitmqctl_escript": attr.label(default = "//deps/rabbitmq_cli:rabbitmqctl"), - "plugins": attr.label_list(), - }, + attrs = RABBITMQ_HOME_ATTRS, ) def _dirname(p): diff --git a/release-notes/3.8.25.md b/release-notes/3.8.25.md new file mode 100644 index 0000000000..892f35bfe1 --- /dev/null +++ b/release-notes/3.8.25.md @@ -0,0 +1,67 @@ +## RabbitMQ 3.8.25 + +RabbitMQ `3.8.25` is a maintenance release. +All users are recommended to upgrade to this release. + +### Obtaining Packages + +RabbitMQ releases are distributed via [GitHub](https://github.com/rabbitmq/rabbitmq-server/releases), [Cloudsmith](https://cloudsmith.io/~rabbitmq/repos/), +and [PackageCloud](https://packagecloud.io/rabbitmq). + +### Erlang/OTP Compatibility Notes + +This release [requires Erlang 23.2](https://www.rabbitmq.com/which-erlang.html) and [supports Erlang 24](https://blog.rabbitmq.com/posts/2021/03/erlang-24-support-roadmap/). + +[Provisioning Latest Erlang Releases](https://www.rabbitmq.com/which-erlang.html#erlang-repositories) explains +what package repositories and tools can be used to provision modern Erlang versions. + + +## Upgrade and Compatibility Notes + +See the [Upgrading guide](https://www.rabbitmq.com/upgrade.html) for general documentation on upgrades and +[RabbitMQ change log](https://www.rabbitmq.com/changelog.html) for release notes of other releases. + +If upgrading from a`3.7.x` release, see [3.8.0 release notes](https://github.com/rabbitmq/rabbitmq-server/releases/tag/v3.8.0) +upgrade and compatibility notes first. + +If upgrading from a `3.6.x` or older [release series](https://www.rabbitmq.com/versions.html), first upgrade +to [`3.7.27`](https://github.com/rabbitmq/rabbitmq-server/releases/tag/v3.7.27) and then to this version. + + +## Getting Help + +Any questions about this release, upgrades or RabbitMQ in general are welcome on the [RabbitMQ mailing list](https://groups.google.com/forum/#!forum/rabbitmq-users) +and [RabbitMQ community Slack](https://rabbitmq-slack.herokuapp.com/). + + +## Changes Worth Mentioning + +Release notes are kept under [rabbitmq-server/release-notes](https://github.com/rabbitmq/rabbitmq-server/tree/v3.8.x/release-notes). +Contributors are encouraged to update them together with their changes. This helps with release automation and more +consistent release schedule. + + +### Prometheus Plugin + +#### Enhancements + + * New Prometheus metrics for alarms: + * `rabbitmq_alarms_file_descriptor_limit` 1|0 + * `rabbitmq_alarms_free_disk_space_watermark` 1|0 + * `rabbitmq_alarms_memory_used_watermark` 1|0 + + While some of the alarms have cluster-wide effect, these metrics are node-local. + + GitHub issue: [#2653](https://github.com/rabbitmq/rabbitmq-server/pull/2653) + + +## Dependency Upgrades + + None in this release. + + +## Source Code Archives + +To obtain source code of the entire distribution, please download the archive named `rabbitmq-server-3.8.25.tar.xz` +instead of the source tarball produced by GitHub. + diff --git a/release-notes/3.8.26.md b/release-notes/3.8.26.md new file mode 100644 index 0000000000..eadaf7cae8 --- /dev/null +++ b/release-notes/3.8.26.md @@ -0,0 +1,64 @@ +## RabbitMQ 3.8.26 + +RabbitMQ `3.8.26` is a maintenance release. +All users are recommended to upgrade to this release. + +### Obtaining Packages + +RabbitMQ releases are distributed via [GitHub](https://github.com/rabbitmq/rabbitmq-server/releases), [Cloudsmith](https://cloudsmith.io/~rabbitmq/repos/), +and [PackageCloud](https://packagecloud.io/rabbitmq). + +### Erlang/OTP Compatibility Notes + +This release [requires Erlang 23.2](https://www.rabbitmq.com/which-erlang.html) and [supports Erlang 24](https://blog.rabbitmq.com/posts/2021/03/erlang-24-support-roadmap/). + +[Provisioning Latest Erlang Releases](https://www.rabbitmq.com/which-erlang.html#erlang-repositories) explains +what package repositories and tools can be used to provision modern Erlang versions. + + +## Upgrade and Compatibility Notes + +See the [Upgrading guide](https://www.rabbitmq.com/upgrade.html) for general documentation on upgrades and +[RabbitMQ change log](https://www.rabbitmq.com/changelog.html) for release notes of other releases. + +If upgrading from a`3.7.x` release, see [3.8.0 release notes](https://github.com/rabbitmq/rabbitmq-server/releases/tag/v3.8.0) +upgrade and compatibility notes first. + +If upgrading from a `3.6.x` or older [release series](https://www.rabbitmq.com/versions.html), first upgrade +to [`3.7.27`](https://github.com/rabbitmq/rabbitmq-server/releases/tag/v3.7.27) and then to this version. + + +## Getting Help + +Any questions about this release, upgrades or RabbitMQ in general are welcome on the [RabbitMQ mailing list](https://groups.google.com/forum/#!forum/rabbitmq-users) +and [RabbitMQ community Slack](https://rabbitmq-slack.herokuapp.com/). + + +## Changes Worth Mentioning + +Release notes are kept under [rabbitmq-server/release-notes](https://github.com/rabbitmq/rabbitmq-server/tree/v3.8.x/release-notes). +Contributors are encouraged to update them together with their changes. This helps with release automation and more +consistent release schedule. + + +### OAuth 2 Plugin + +#### Bug Fixes + + * Signing keys specified in `rabbitmq.conf` were not translated correctly, + resulting in exceptions during permission checks. + + GitHub issue: [#3759](https://github.com/rabbitmq/rabbitmq-server/pull/3759) + + + +## Dependency Upgrades + + None in this release. + + +## Source Code Archives + +To obtain source code of the entire distribution, please download the archive named `rabbitmq-server-3.8.26.tar.xz` +instead of the source tarball produced by GitHub. + diff --git a/release-notes/3.9.10.md b/release-notes/3.9.10.md index 7e35dbf8a7..45243dbe90 100644 --- a/release-notes/3.9.10.md +++ b/release-notes/3.9.10.md @@ -2,7 +2,7 @@ RabbitMQ `3.9.10` is a maintenance release in the `3.9.x` release series. Please refer to the **Upgrading to 3.9** section from [v3.9.0 release notes](https://github.com/rabbitmq/rabbitmq-server/releases/tag/v3.9.0) if upgrading from a version prior to 3.9.0. -This release requires at least Erlang 23.2, and supports the latest Erlang 24 version, 24.1.2 at the time of release. [RabbitMQ and Erlang/OTP Compatibility Matrix](https://www.rabbitmq.com/which-erlang.html) has more details on Erlang version requirements for RabbitMQ. +This release requires at least Erlang 23.2, and supports Erlang 24. [RabbitMQ and Erlang/OTP Compatibility Matrix](https://www.rabbitmq.com/which-erlang.html) has more details on Erlang version requirements for RabbitMQ. @@ -34,10 +34,19 @@ Contributors are encouraged to update them together with their changes. This hel GitHub issue: [#3739](https://github.com/rabbitmq/rabbitmq-server/issues/3739) +### OAuth 2 Plugin + +#### Bug Fixes + + * Signing keys specified in `rabbitmq.conf` were not translated correctly, + resulting in exceptions during permission checks. + + GitHub issue: [#3759](https://github.com/rabbitmq/rabbitmq-server/pull/3759) + ## Dependency Upgrades -None in this release. +* Ra was [upgraded to `2.0.3`](https://github.com/rabbitmq/ra/compare/v2.0.2...v2.0.3) ## Source Code Archives |