feat (prometheus): metrics for individual message queues

The `_prometheus` endpoint today includes size/min/max metrics
across all message queues. This adds a new metric -
`erlang_message_queue_size{queue_name="<name>"}` which tracks the
size of individual message queues.

This could replace the previous metrics since those can be derived from
the new metric by prometheus, but I've left them in place for
compatibility.

3 files changed, 16 insertions, 9 deletions

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index a14b7b1e6..5e577d96d 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -26,4 +26,4 @@
     "postCreateCommand": "./configure && make",
 
     "extensions": ["erlang-ls.erlang-ls"]
-}
+}
\ No newline at end of file
diff --git a/src/couch_prometheus/src/couch_prometheus_server.erl b/src/couch_prometheus/src/couch_prometheus_server.erl
index 70c4790ab..939ce00cf 100644
--- a/src/couch_prometheus/src/couch_prometheus_server.erl
+++ b/src/couch_prometheus/src/couch_prometheus_server.erl
@@ -177,24 +177,27 @@ get_io_stats() ->
     ].
 
 get_message_queue_stats() ->
-    QLenFun = fun(Name) -> message_queue_len(whereis(Name)) end,
-    Queues = lists:map(QLenFun, registered()),
+    QFun = fun(Name) -> {Name, message_queue_len(whereis(Name))} end,
+    Queues = lists:map(QFun, registered()),
+    QueueLens = lists:map(fun({_, Len}) -> Len end, Queues),
+    QueueLenByLabel = lists:map(fun({Name, Len}) -> {[{queue_name, Name}], Len} end, Queues),
     [
         to_prom(
-            erlang_message_queues, gauge, "total size of all message queues", lists:sum(Queues)
+            erlang_message_queues, gauge, "total size of all message queues", lists:sum(QueueLens)
         ),
         to_prom(
             erlang_message_queue_min,
             gauge,
             "minimum size across all message queues",
-            lists:min(Queues)
+            lists:min(QueueLens)
         ),
         to_prom(
             erlang_message_queue_max,
             gauge,
             "maximum size across all message queues",
-            lists:max(Queues)
-        )
+            lists:max(QueueLens)
+        ),
+        to_prom(erlang_message_queue_size, gauge, "size of message queue", QueueLenByLabel)
     ].
 
 message_queue_len(undefined) ->
diff --git a/src/couch_prometheus/src/couch_prometheus_util.erl b/src/couch_prometheus/src/couch_prometheus_util.erl
index 9e4a74e49..16886f4a7 100644
--- a/src/couch_prometheus/src/couch_prometheus_util.erl
+++ b/src/couch_prometheus/src/couch_prometheus_util.erl
@@ -109,9 +109,13 @@ type_def(Metric, Type, Desc) ->
         to_bin(io_lib:format("# TYPE ~s ~s", [Name, Type]))
     ].
 
-to_prom(Metric, Type, Desc, Data) ->
+% support creating a metric series with multiple label/values.
+% Instances is of the form [{[{LabelName, LabelValue}], Value}, ...]
+to_prom(Metric, Type, Desc, Instances) when is_list(Instances) ->
     TypeStr = type_def(Metric, Type, Desc),
-    [TypeStr] ++ to_prom(Metric, Data).
+    [TypeStr] ++ lists:flatmap(fun(Inst) -> to_prom(Metric, Inst) end, Instances);
+to_prom(Metric, Type, Desc, Data) ->
+    to_prom(Metric, Type, Desc, [Data]).
 
 to_prom(Metric, Instances) when is_list(Instances) ->
     lists:flatmap(fun(Inst) -> to_prom(Metric, Inst) end, Instances);