Add /metrics/per_object endpoint

Regardless of the value of `return_per_object_metrics`, this endpoint
always returns per-object metrics. This allows scraping both endpoints
at different intervals or scraping per-object metrics only during
debugging.

Co-authored-by: Mirah Gary <mgary@vmware.com>

4 files changed, 54 insertions, 13 deletions

diff --git a/deps/rabbitmq_prometheus/src/collectors/prometheus_rabbitmq_core_metrics_collector.erl b/deps/rabbitmq_prometheus/src/collectors/prometheus_rabbitmq_core_metrics_collector.erl
index 255260627a..724a50ee13 100644
--- a/deps/rabbitmq_prometheus/src/collectors/prometheus_rabbitmq_core_metrics_collector.erl
+++ b/deps/rabbitmq_prometheus/src/collectors/prometheus_rabbitmq_core_metrics_collector.erl
@@ -221,8 +221,13 @@ register() ->
 
 deregister_cleanup(_) -> ok.
 
+collect_mf(per_object, Callback) ->
+    collect(true, Callback);
 collect_mf(_Registry, Callback) ->
-    {ok, PerObjectMetrics} = application:get_env(rabbitmq_prometheus, return_per_object_metrics),
+    PerObjectMetrics = application:get_env(rabbitmq_prometheus, return_per_object_metrics, false),
+    collect(PerObjectMetrics, Callback).
+
+collect(PerObjectMetrics, Callback) ->
     [begin
          Data = get_data(Table, PerObjectMetrics),
          mf(Callback, Contents, Data)
diff --git a/deps/rabbitmq_prometheus/src/rabbit_prometheus_dispatcher.erl b/deps/rabbitmq_prometheus/src/rabbit_prometheus_dispatcher.erl
index 910123c005..08fc2d7854 100644
--- a/deps/rabbitmq_prometheus/src/rabbit_prometheus_dispatcher.erl
+++ b/deps/rabbitmq_prometheus/src/rabbit_prometheus_dispatcher.erl
@@ -14,11 +14,21 @@
 build_dispatcher() ->
     {ok, _} = application:ensure_all_started(prometheus),
     prometheus_registry:register_collectors([prometheus_rabbitmq_core_metrics_collector]),
+    prometheus_registry:register_collectors(per_object, [
+        prometheus_vm_system_info_collector,
+        prometheus_vm_dist_collector,
+        prometheus_vm_memory_collector,
+        prometheus_mnesia_collector,
+        prometheus_vm_statistics_collector,
+        prometheus_vm_msacc_collector,
+        prometheus_rabbitmq_core_metrics_collector
+        ]),
     rabbit_prometheus_handler:setup(),
     cowboy_router:compile([{'_', dispatcher()}]).
 
 dispatcher() ->
-    [{path() ++ "/[:registry]", rabbit_prometheus_handler, []}].
+    [{path() ++ "/[:registry]", rabbit_prometheus_handler, []},
+    {path() ++ "/[:per_object]", rabbit_prometheus_handler, []}].
 
 path() ->
     application:get_env(rabbitmq_prometheus, path, ?DEFAULT_PATH).
diff --git a/deps/rabbitmq_prometheus/src/rabbit_prometheus_handler.erl b/deps/rabbitmq_prometheus/src/rabbit_prometheus_handler.erl
index bb7a98150d..c453749724 100644
--- a/deps/rabbitmq_prometheus/src/rabbit_prometheus_handler.erl
+++ b/deps/rabbitmq_prometheus/src/rabbit_prometheus_handler.erl
@@ -31,20 +31,22 @@ is_authorized(ReqData, Context) ->
     {true, ReqData, Context}.
 
 setup() ->
-    TelemetryRegistry = telemetry_registry(),
+    setup_metrics(telemetry_registry()),
+    setup_metrics(per_object).
 
+setup_metrics(Registry) ->
     ScrapeDuration = [{name, ?SCRAPE_DURATION},
                       {help, "Scrape duration"},
                       {labels, ["registry", "content_type"]},
-                      {registry, TelemetryRegistry}],
+                      {registry, Registry}],
     ScrapeSize = [{name, ?SCRAPE_SIZE},
                   {help, "Scrape size, not encoded"},
                   {labels, ["registry", "content_type"]},
-                  {registry, TelemetryRegistry}],
+                  {registry, Registry}],
     ScrapeEncodedSize = [{name, ?SCRAPE_ENCODED_SIZE},
                          {help, "Scrape size, encoded"},
                          {labels, ["registry", "content_type", "encoding"]},
-                         {registry, TelemetryRegistry}],
+                         {registry, Registry}],
 
     prometheus_summary:declare(ScrapeDuration),
     prometheus_summary:declare(ScrapeSize),
@@ -106,14 +108,14 @@ format_metrics(Request, Registry) ->
     encode_format(ContentType, binary_to_list(Encoding), Scrape, Registry).
 
 render_format(ContentType, Registry) ->
-    TelemetryRegistry = telemetry_registry(),
+    %TelemetryRegistry = telemetry_registry(),
 
     Scrape = prometheus_summary:observe_duration(
-               TelemetryRegistry,
+               Registry,
                ?SCRAPE_DURATION,
                [Registry, ContentType],
                fun () -> prometheus_text_format:format(Registry) end),
-    prometheus_summary:observe(TelemetryRegistry,
+    prometheus_summary:observe(Registry,
                                ?SCRAPE_SIZE,
                                [Registry, ContentType],
                                iolist_size(Scrape)),
diff --git a/deps/rabbitmq_prometheus/test/rabbit_prometheus_http_SUITE.erl b/deps/rabbitmq_prometheus/test/rabbit_prometheus_http_SUITE.erl
index 9ef4a43efa..634cbc10bb 100644
--- a/deps/rabbitmq_prometheus/test/rabbit_prometheus_http_SUITE.erl
+++ b/deps/rabbitmq_prometheus/test/rabbit_prometheus_http_SUITE.erl
@@ -21,6 +21,7 @@ all() ->
         {group, config_port},
         {group, aggregated_metrics},
         {group, per_object_metrics},
+        {group, per_object_endpoint_metrics},
         {group, commercial}
     ].
 
@@ -34,7 +35,11 @@ groups() ->
             specific_erlang_metrics_present_test
         ]},
         {per_object_metrics, [], [
-            per_object_metrics_test,
+            globally_configure_per_object_metrics_test,
+            specific_erlang_metrics_present_test
+        ]},
+        {per_object_endpoint_metrics, [], [
+            endpoint_per_object_metrics,
             specific_erlang_metrics_present_test
         ]},
         {commercial, [], [
@@ -69,6 +74,12 @@ init_per_group(per_object_metrics, Config0) ->
     PathConfig = {rabbitmq_prometheus, [{return_per_object_metrics, true}]},
     Config1 = rabbit_ct_helpers:merge_app_env(Config0, PathConfig),
     init_per_group(aggregated_metrics, Config1);
+init_per_group(per_object_endpoint_metrics, Config0) ->
+    PathConfig = {rabbitmq_prometheus, [
+        {return_per_object_metrics, false}
+    ]},
+    Config1 = rabbit_ct_helpers:merge_app_env(Config0, PathConfig),
+    init_per_group(aggregated_metrics, Config1);
 init_per_group(aggregated_metrics, Config0) ->
     Config1 = rabbit_ct_helpers:merge_app_env(
         Config0,
@@ -214,8 +225,14 @@ aggregated_metrics_test(Config) ->
     %% Check raft_entry_commit_latency_seconds because we are aggregating it
     ?assertEqual(match, re:run(Body, "^rabbitmq_raft_entry_commit_latency_seconds ", [{capture, none}, multiline])).
 
-per_object_metrics_test(Config) ->
-    {_Headers, Body} = http_get_with_pal(Config, [], 200),
+endpoint_per_object_metrics(Config) ->
+    per_object_metrics_test(Config, "/metrics/per_object").
+
+globally_configure_per_object_metrics_test(Config) ->
+    per_object_metrics_test(Config, "/metrics").
+
+per_object_metrics_test(Config, Path) ->
+    {_Headers, Body} = http_get_with_pal(Config, Path, [], 200),
     ?assertEqual(match, re:run(Body, "^# TYPE", [{capture, none}, multiline])),
     ?assertEqual(match, re:run(Body, "^# HELP", [{capture, none}, multiline])),
     ?assertEqual(match, re:run(Body, ?config(queue_name, Config), [{capture, none}])),
@@ -268,6 +285,9 @@ specific_erlang_metrics_present_test(Config) ->
 
 http_get(Config, ReqHeaders, CodeExp) ->
     Path = proplists:get_value(prometheus_path, Config, "/metrics"),
+    http_get(Config, Path, ReqHeaders, CodeExp).
+
+http_get(Config, Path, ReqHeaders, CodeExp) ->
     Port = proplists:get_value(prometheus_port, Config, 15692),
     URI = lists:flatten(io_lib:format("http://localhost:~p~s", [Port, Path])),
     {ok, {{_HTTP, CodeAct, _}, Headers, Body}} =
@@ -276,7 +296,11 @@ http_get(Config, ReqHeaders, CodeExp) ->
     {Headers, Body}.
 
 http_get_with_pal(Config, ReqHeaders, CodeExp) ->
-    {Headers, Body} = http_get(Config, ReqHeaders, CodeExp),
+    Path = proplists:get_value(prometheus_path, Config, "/metrics"),
+    http_get_with_pal(Config, Path, ReqHeaders, CodeExp).
+
+http_get_with_pal(Config, Path, ReqHeaders, CodeExp) ->
+    {Headers, Body} = http_get(Config, Path, ReqHeaders, CodeExp),
     %% Print and log response body - it makes is easier to find why a match failed
     ct:pal(Body),
     {Headers, Body}.