Add report logging (#4483)

Add new report logging mechanism to log a map of key/value pairs
---------

Co-authored-by: ILYA Khlopotov <iilyak@apache.org>

4 files changed, 41 insertions, 15 deletions

diff --git a/src/mango/src/mango_cursor_nouveau.erl b/src/mango/src/mango_cursor_nouveau.erl
index 8a6525cca..7cdad20cd 100644
--- a/src/mango/src/mango_cursor_nouveau.erl
+++ b/src/mango/src/mango_cursor_nouveau.erl
@@ -122,10 +122,10 @@ execute(Cursor, UserFun, UserAcc) ->
             JsonBM = nouveau_bookmark:pack(FinalBM),
             Arg = {add_key, bookmark, JsonBM},
             {_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
-            FinalUserAcc0 = mango_execution_stats:maybe_add_stats(
+            {FinalUserAcc0, Stats1} = mango_execution_stats:maybe_add_stats(
                 Opts, UserFun, Stats0, FinalUserAcc
             ),
-            FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0),
+            FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats1, FinalUserAcc0),
             {ok, FinalUserAcc1}
     end.
 
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index 959603316..7454aebfd 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -127,10 +127,12 @@ execute(Cursor, UserFun, UserAcc) ->
             JsonBM = dreyfus_bookmark:pack(FinalBM),
             Arg = {add_key, bookmark, JsonBM},
             {_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
-            FinalUserAcc0 = mango_execution_stats:maybe_add_stats(
+            {FinalUserAcc0, Stats1} = mango_execution_stats:maybe_add_stats(
                 Opts, UserFun, Stats0, FinalUserAcc
             ),
-            FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0),
+            %% This needs Stats1 as log_end is called in maybe_add_stats
+            mango_execution_stats:log_stats(Stats1),
+            FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats1, FinalUserAcc0),
             {ok, FinalUserAcc1}
     end.
 
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index eec8dc4fe..d5cffbc5c 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -229,11 +229,13 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
                     Arg = {add_key, bookmark, NewBookmark},
                     {_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc),
                     Stats0 = LastCursor#cursor.execution_stats,
-                    FinalUserAcc0 = mango_execution_stats:maybe_add_stats(
+                    {FinalUserAcc0, Stats1} = mango_execution_stats:maybe_add_stats(
                         Opts, UserFun, Stats0, FinalUserAcc
                     ),
+                    %% This needs Stats1 as log_end is called in maybe_add_stats
+                    mango_execution_stats:log_stats(Stats1),
                     FinalUserAcc1 = mango_cursor:maybe_add_warning(
-                        UserFun, Cursor, Stats0, FinalUserAcc0
+                        UserFun, Cursor, Stats1, FinalUserAcc0
                     ),
                     {ok, FinalUserAcc1};
                 {error, Reason} ->
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
index 0db3edf5f..66104e89e 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -14,6 +14,7 @@
 
 -export([
     to_json/1,
+    to_map/1,
     incr_keys_examined/1,
     incr_docs_examined/1,
     incr_docs_examined/2,
@@ -21,6 +22,7 @@
     incr_results_returned/1,
     log_start/1,
     log_end/1,
+    log_stats/1,
     maybe_add_stats/4
 ]).
 
@@ -35,6 +37,15 @@ to_json(Stats) ->
         {execution_time_ms, Stats#execution_stats.executionTimeMs}
     ]}.
 
+to_map(Stats) ->
+    #{
+        total_keys_examined => Stats#execution_stats.totalKeysExamined,
+        total_docs_examined => Stats#execution_stats.totalDocsExamined,
+        total_quorum_docs_examined => Stats#execution_stats.totalQuorumDocsExamined,
+        results_returned => Stats#execution_stats.resultsReturned,
+        execution_time_ms => Stats#execution_stats.executionTimeMs
+    }.
+
 incr_keys_examined(Stats) ->
     Stats#execution_stats{
         totalKeysExamined = Stats#execution_stats.totalKeysExamined + 1
@@ -74,13 +85,24 @@ log_end(Stats) ->
 maybe_add_stats(Opts, UserFun, Stats0, UserAcc) ->
     Stats1 = log_end(Stats0),
     couch_stats:update_histogram([mango, query_time], Stats1#execution_stats.executionTimeMs),
+    %% TODO: add rows read when we collect the stats
+    %% TODO: add docs vs quorum docs
+    chttpd_stats:incr_reads(Stats1#execution_stats.totalDocsExamined),
+
+    FinalAcc =
+        case couch_util:get_value(execution_stats, Opts) of
+            true ->
+                JSONValue = to_json(Stats1),
+                Arg = {add_key, execution_stats, JSONValue},
+                {_Go, FinalUserAcc} = UserFun(Arg, UserAcc),
+                FinalUserAcc;
+            _ ->
+                UserAcc
+        end,
+    {FinalAcc, Stats1}.
 
-    case couch_util:get_value(execution_stats, Opts) of
-        true ->
-            JSONValue = to_json(Stats1),
-            Arg = {add_key, execution_stats, JSONValue},
-            {_Go, FinalUserAcc} = UserFun(Arg, UserAcc),
-            FinalUserAcc;
-        _ ->
-            UserAcc
-    end.
+log_stats(Stats) ->
+    MStats0 = to_map(Stats),
+    Nonce = list_to_binary(couch_log_util:get_msg_id()),
+    MStats1 = MStats0#{nonce => Nonce},
+    couch_log:report("mango-stats", MStats1).