summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/docs/src/api/database/find.rst3
-rw-r--r--src/mango/src/mango.hrl10
-rw-r--r--src/mango/src/mango_cursor_view.erl175
-rw-r--r--src/mango/src/mango_execution_stats.erl42
-rw-r--r--src/mango/test/15-execution-stats-test.py17
5 files changed, 219 insertions, 28 deletions
diff --git a/src/docs/src/api/database/find.rst b/src/docs/src/api/database/find.rst
index 027ddf8ee..ede5598c9 100644
--- a/src/docs/src/api/database/find.rst
+++ b/src/docs/src/api/database/find.rst
@@ -145,7 +145,7 @@ Example response when finding documents using an index:
}
],
"execution_stats": {
- "total_keys_examined": 0,
+ "total_keys_examined": 200,
"total_docs_examined": 200,
"total_quorum_docs_examined": 0,
"results_returned": 2,
@@ -925,7 +925,6 @@ The execution statistics currently include:
| Field | Description |
+================================+============================================+
| ``total_keys_examined`` | Number of index keys examined. |
-| | Currently always 0. |
+--------------------------------+--------------------------------------------+
| ``total_docs_examined`` | Number of documents fetched from the |
| | database / index, equivalent to using |
diff --git a/src/mango/src/mango.hrl b/src/mango/src/mango.hrl
index 2ff07aa4b..d8fa095bf 100644
--- a/src/mango/src/mango.hrl
+++ b/src/mango/src/mango.hrl
@@ -30,6 +30,14 @@
-type selector() :: any().
-type ejson() :: {[{atom(), any()}]}.
--type shard_stats() :: {docs_examined, non_neg_integer()}.
+-type shard_stats() :: shard_stats_v1() | shard_stats_v2().
+
+-type shard_stats_v1() :: {docs_examined, non_neg_integer()}.
+-type shard_stats_v2() ::
+ #{
+ docs_examined => non_neg_integer(),
+ keys_examined => non_neg_integer()
+ }.
+
-type row_property_key() :: id | key | value | doc.
-type row_properties() :: [{row_property_key(), any()}].
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index d5cffbc5c..e044c56fc 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -79,6 +79,15 @@ viewcbargs_get(fields, Args) when is_map(Args) ->
viewcbargs_get(covering_index, Args) when is_map(Args) ->
maps:get(covering_index, Args, undefined).
+-spec shard_stats_get(Key, Args) -> Stat when
+ Key :: docs_examined | keys_examined,
+ Args :: shard_stats_v2(),
+ Stat :: non_neg_integer().
+shard_stats_get(docs_examined, Args) when is_map(Args) ->
+ maps:get(docs_examined, Args, 0);
+shard_stats_get(keys_examined, Args) when is_map(Args) ->
+ maps:get(keys_examined, Args, 0).
+
-spec create(Db, Indexes, Selector, Options) -> {ok, #cursor{}} when
Db :: database(),
Indexes :: [#idx{}],
@@ -187,7 +196,12 @@ base_args(#cursor{index = Idx, selector = Selector, fields = Fields} = Cursor) -
{selector, Selector},
{callback_args, viewcbargs_new(Selector, Fields, undefined)},
- {ignore_partition_query_limit, true}
+ {ignore_partition_query_limit, true},
+
+ % Request execution statistics in a map. The purpose of this option is
+ % to maintain interoperability on version upgrades.
+ % TODO remove this option in a later version.
+ {execution_stats_map, true}
]
}.
@@ -326,11 +340,13 @@ choose_best_index(IndexRanges) ->
(ok, ddoc_updated) -> any().
view_cb({meta, Meta}, Acc) ->
% Map function starting
- put(mango_docs_examined, 0),
+ mango_execution_stats:shard_init(),
set_mango_msg_timestamp(),
ok = rexi:stream2({meta, Meta}),
{ok, Acc};
view_cb({row, Row}, #mrargs{extra = Options} = Acc) ->
+ mango_execution_stats:shard_incr_keys_examined(),
+ couch_stats:increment_counter([mango, keys_examined]),
ViewRow = #view_row{
id = couch_util:get_value(id, Row),
key = couch_util:get_value(key, Row),
@@ -379,14 +395,23 @@ view_cb({row, Row}, #mrargs{extra = Options} = Acc) ->
ok = rexi:stream2(ViewRow),
set_mango_msg_timestamp();
{Doc, _} ->
- put(mango_docs_examined, get(mango_docs_examined) + 1),
+ mango_execution_stats:shard_incr_docs_examined(),
couch_stats:increment_counter([mango, docs_examined]),
Process(Doc)
end,
{ok, Acc};
-view_cb(complete, Acc) ->
+view_cb(complete, #mrargs{extra = Options} = Acc) ->
+ ShardStats = mango_execution_stats:shard_get_stats(),
+ Stats =
+ case couch_util:get_value(execution_stats_map, Options, false) of
+ true ->
+ ShardStats;
+ false ->
+ DocsExamined = maps:get(docs_examined, ShardStats),
+ {docs_examined, DocsExamined}
+ end,
% Send shard-level execution stats
- ok = rexi:stream2({execution_stats, {docs_examined, get(mango_docs_examined)}}),
+ ok = rexi:stream2({execution_stats, Stats}),
% Finish view output
ok = rexi:stream_last(complete),
{ok, Acc};
@@ -459,12 +484,20 @@ handle_message({row, Props}, Cursor) ->
couch_log:error("~s :: Error loading doc: ~p", [?MODULE, Error]),
{ok, Cursor}
end;
-handle_message({execution_stats, ShardStats}, #cursor{execution_stats = Stats} = Cursor) ->
- {docs_examined, DocsExamined} = ShardStats,
- Cursor1 = Cursor#cursor{
+handle_message({execution_stats, {docs_examined, DocsExamined}}, Cursor0) ->
+ #cursor{execution_stats = Stats} = Cursor0,
+ Cursor = Cursor0#cursor{
execution_stats = mango_execution_stats:incr_docs_examined(Stats, DocsExamined)
},
- {ok, Cursor1};
+ {ok, Cursor};
+handle_message({execution_stats, #{} = ShardStats}, Cursor0) ->
+ DocsExamined = shard_stats_get(docs_examined, ShardStats),
+ KeysExamined = shard_stats_get(keys_examined, ShardStats),
+ #cursor{execution_stats = Stats0} = Cursor0,
+ Stats1 = mango_execution_stats:incr_docs_examined(Stats0, DocsExamined),
+ Stats = mango_execution_stats:incr_keys_examined(Stats1, KeysExamined),
+ Cursor = Cursor0#cursor{execution_stats = Stats},
+ {ok, Cursor};
handle_message(complete, Cursor) ->
{ok, Cursor};
handle_message({error, Reason}, _Cursor) ->
@@ -702,7 +735,8 @@ base_opts_test() ->
fields => Fields,
covering_index => undefined
}},
- {ignore_partition_query_limit, true}
+ {ignore_partition_query_limit, true},
+ {execution_stats_map, true}
],
MRArgs =
#mrargs{
@@ -945,6 +979,7 @@ execute_test_() ->
[
?TDEF_FE(t_execute_empty),
?TDEF_FE(t_execute_ok_all_docs),
+ ?TDEF_FE(t_execute_ok_all_docs_with_execution_stats),
?TDEF_FE(t_execute_ok_query_view),
?TDEF_FE(t_execute_error)
]
@@ -997,7 +1032,8 @@ t_execute_ok_all_docs(_) ->
fields => Fields,
covering_index => undefined
}},
- {ignore_partition_query_limit, true}
+ {ignore_partition_query_limit, true},
+ {execution_stats_map, true}
],
Args =
#mrargs{
@@ -1060,7 +1096,8 @@ t_execute_ok_query_view(_) ->
fields => Fields,
covering_index => undefined
}},
- {ignore_partition_query_limit, true}
+ {ignore_partition_query_limit, true},
+ {execution_stats_map, true}
],
Args =
#mrargs{
@@ -1084,6 +1121,79 @@ t_execute_ok_query_view(_) ->
?assertEqual({ok, updated_accumulator}, execute(Cursor, fun foo:bar/2, accumulator)),
?assert(meck:called(fabric, query_view, '_')).
+t_execute_ok_all_docs_with_execution_stats(_) ->
+ Bookmark = bookmark,
+ Stats =
+ {[
+ {total_keys_examined, 0},
+ {total_docs_examined, 0},
+ {total_quorum_docs_examined, 0},
+ {results_returned, 0},
+ {execution_time_ms, '_'}
+ ]},
+ UserFnDefinition =
+ [
+ {[{add_key, bookmark, Bookmark}, accumulator], {undefined, updated_accumulator1}},
+ {
+ [{add_key, execution_stats, Stats}, updated_accumulator1],
+ {undefined, updated_accumulator2}
+ }
+ ],
+ meck:expect(foo, bar, UserFnDefinition),
+ Index = #idx{type = <<"json">>, def = all_docs},
+ Selector = {[]},
+ Fields = all_fields,
+ Cursor =
+ #cursor{
+ index = Index,
+ db = db,
+ selector = Selector,
+ fields = Fields,
+ ranges = [{'$gte', start_key, '$lte', end_key}],
+ opts = [{user_ctx, user_ctx}, {execution_stats, true}],
+ bookmark = nil
+ },
+ Cursor1 =
+ Cursor#cursor{
+ user_acc = accumulator,
+ user_fun = fun foo:bar/2,
+ execution_stats = '_'
+ },
+ Cursor2 =
+ Cursor1#cursor{
+ bookmark = Bookmark,
+ bookmark_docid = undefined,
+ bookmark_key = undefined,
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
+ },
+ Extra =
+ [
+ {callback, {mango_cursor_view, view_cb}},
+ {selector, Selector},
+ {callback_args, #{
+ selector => Selector,
+ fields => Fields,
+ covering_index => undefined
+ }},
+ {ignore_partition_query_limit, true},
+ {execution_stats_map, true}
+ ],
+ Args =
+ #mrargs{
+ view_type = map,
+ reduce = false,
+ start_key = [start_key],
+ end_key = [end_key, ?MAX_JSON_OBJ],
+ include_docs = true,
+ extra = Extra
+ },
+ Parameters = [
+ db, [{user_ctx, user_ctx}], fun mango_cursor_view:handle_all_docs_message/2, Cursor1, Args
+ ],
+ meck:expect(fabric, all_docs, Parameters, meck:val({ok, Cursor2})),
+ ?assertEqual({ok, updated_accumulator2}, execute(Cursor, fun foo:bar/2, accumulator)),
+ ?assert(meck:called(fabric, all_docs, '_')).
+
t_execute_error(_) ->
Cursor =
#cursor{
@@ -1119,7 +1229,8 @@ view_cb_test_() ->
?TDEF_FE(t_view_cb_row_matching_covered_doc),
?TDEF_FE(t_view_cb_row_non_matching_covered_doc),
?TDEF_FE(t_view_cb_row_backwards_compatible),
- ?TDEF_FE(t_view_cb_complete),
+ ?TDEF_FE(t_view_cb_complete_shard_stats_v1),
+ ?TDEF_FE(t_view_cb_complete_shard_stats_v2),
?TDEF_FE(t_view_cb_ok)
]
}.
@@ -1143,7 +1254,7 @@ t_view_cb_row_matching_regular_doc(_) ->
}}
]
},
- put(mango_docs_examined, 0),
+ mango_execution_stats:shard_init(),
?assertEqual({ok, Accumulator}, view_cb({row, Row}, Accumulator)),
?assert(meck:called(rexi, stream2, '_')).
@@ -1161,7 +1272,7 @@ t_view_cb_row_non_matching_regular_doc(_) ->
}}
]
},
- put(mango_docs_examined, 0),
+ mango_execution_stats:shard_init(),
put(mango_last_msg_timestamp, os:timestamp()),
?assertEqual({ok, Accumulator}, view_cb({row, Row}, Accumulator)),
?assertNot(meck:called(rexi, stream2, '_')).
@@ -1179,6 +1290,7 @@ t_view_cb_row_null_doc(_) ->
}}
]
},
+ mango_execution_stats:shard_init(),
put(mango_last_msg_timestamp, os:timestamp()),
?assertEqual({ok, Accumulator}, view_cb({row, Row}, Accumulator)),
?assertNot(meck:called(rexi, stream2, '_')).
@@ -1197,6 +1309,7 @@ t_view_cb_row_missing_doc_triggers_quorum_fetch(_) ->
}}
]
},
+ mango_execution_stats:shard_init(),
?assertEqual({ok, Accumulator}, view_cb({row, Row}, Accumulator)),
?assert(meck:called(rexi, stream2, '_')).
@@ -1222,6 +1335,7 @@ t_view_cb_row_matching_covered_doc(_) ->
}}
]
},
+ mango_execution_stats:shard_init(),
?assertEqual({ok, Accumulator}, view_cb({row, Row}, Accumulator)),
?assert(meck:called(rexi, stream2, '_')).
@@ -1244,6 +1358,7 @@ t_view_cb_row_non_matching_covered_doc(_) ->
}}
]
},
+ mango_execution_stats:shard_init(),
put(mango_last_msg_timestamp, os:timestamp()),
?assertEqual({ok, Accumulator}, view_cb({row, Row}, Accumulator)),
?assertNot(meck:called(rexi, stream2, '_')).
@@ -1252,14 +1367,27 @@ t_view_cb_row_backwards_compatible(_) ->
Row = [{id, id}, {key, key}, {doc, null}],
meck:expect(rexi, stream2, ['_'], undefined),
Accumulator = #mrargs{extra = [{selector, {[]}}]},
+ mango_execution_stats:shard_init(),
put(mango_last_msg_timestamp, os:timestamp()),
?assertEqual({ok, Accumulator}, view_cb({row, Row}, Accumulator)),
?assertNot(meck:called(rexi, stream2, '_')).
-t_view_cb_complete(_) ->
+t_view_cb_complete_shard_stats_v1(_) ->
meck:expect(rexi, stream2, [{execution_stats, {docs_examined, '_'}}], meck:val(ok)),
meck:expect(rexi, stream_last, [complete], meck:val(ok)),
- ?assertEqual({ok, accumulator}, view_cb(complete, accumulator)),
+ Accumulator = #mrargs{},
+ mango_execution_stats:shard_init(),
+ ?assertEqual({ok, Accumulator}, view_cb(complete, Accumulator)),
+ ?assert(meck:called(rexi, stream2, '_')),
+ ?assert(meck:called(rexi, stream_last, '_')).
+
+t_view_cb_complete_shard_stats_v2(_) ->
+ ShardStats = #{docs_examined => '_', keys_examined => '_'},
+ meck:expect(rexi, stream2, [{execution_stats, ShardStats}], meck:val(ok)),
+ meck:expect(rexi, stream_last, [complete], meck:val(ok)),
+ Accumulator = #mrargs{extra = [{execution_stats_map, true}]},
+ mango_execution_stats:shard_init(),
+ ?assertEqual({ok, Accumulator}, view_cb(complete, Accumulator)),
?assert(meck:called(rexi, stream2, '_')),
?assert(meck:called(rexi, stream_last, '_')).
@@ -1323,7 +1451,8 @@ handle_message_test_() ->
?TDEF_FE(t_handle_message_row_ok_triggers_quorum_fetch_no_match),
?TDEF_FE(t_handle_message_row_no_match),
?TDEF_FE(t_handle_message_row_error),
- ?TDEF_FE(t_handle_message_execution_stats),
+ ?TDEF_FE(t_handle_message_execution_stats_v1),
+ ?TDEF_FE(t_handle_message_execution_stats_v2),
?TDEF_FE(t_handle_message_complete),
?TDEF_FE(t_handle_message_error)
]
@@ -1455,7 +1584,7 @@ t_handle_message_row_error(_) ->
meck:delete(mango_util, defer, 3),
meck:delete(couch_log, error, 2).
-t_handle_message_execution_stats(_) ->
+t_handle_message_execution_stats_v1(_) ->
ShardStats = {docs_examined, 42},
ExecutionStats = #execution_stats{totalDocsExamined = 11},
ExecutionStats1 = #execution_stats{totalDocsExamined = 53},
@@ -1463,6 +1592,14 @@ t_handle_message_execution_stats(_) ->
Cursor1 = #cursor{execution_stats = ExecutionStats1},
?assertEqual({ok, Cursor1}, handle_message({execution_stats, ShardStats}, Cursor)).
+t_handle_message_execution_stats_v2(_) ->
+ ShardStats = #{docs_examined => 42, keys_examined => 53},
+ ExecutionStats = #execution_stats{totalDocsExamined = 11, totalKeysExamined = 22},
+ ExecutionStats1 = #execution_stats{totalDocsExamined = 53, totalKeysExamined = 75},
+ Cursor = #cursor{execution_stats = ExecutionStats},
+ Cursor1 = #cursor{execution_stats = ExecutionStats1},
+ ?assertEqual({ok, Cursor1}, handle_message({execution_stats, ShardStats}, Cursor)).
+
t_handle_message_complete(_) ->
?assertEqual({ok, cursor}, handle_message(complete, cursor)).
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
index 66104e89e..350b58bda 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -15,7 +15,7 @@
-export([
to_json/1,
to_map/1,
- incr_keys_examined/1,
+ incr_keys_examined/2,
incr_docs_examined/1,
incr_docs_examined/2,
incr_quorum_docs_examined/1,
@@ -23,11 +23,18 @@
log_start/1,
log_end/1,
log_stats/1,
- maybe_add_stats/4
+ maybe_add_stats/4,
+ shard_init/0,
+ shard_incr_keys_examined/0,
+ shard_incr_docs_examined/0,
+ shard_get_stats/0
]).
+-include("mango.hrl").
-include("mango_cursor.hrl").
+-define(SHARD_STATS_KEY, mango_shard_execution_stats).
+
to_json(Stats) ->
{[
{total_keys_examined, Stats#execution_stats.totalKeysExamined},
@@ -46,9 +53,9 @@ to_map(Stats) ->
execution_time_ms => Stats#execution_stats.executionTimeMs
}.
-incr_keys_examined(Stats) ->
+incr_keys_examined(Stats, N) ->
Stats#execution_stats{
- totalKeysExamined = Stats#execution_stats.totalKeysExamined + 1
+ totalKeysExamined = Stats#execution_stats.totalKeysExamined + N
}.
incr_docs_examined(Stats) ->
@@ -106,3 +113,30 @@ log_stats(Stats) ->
Nonce = list_to_binary(couch_log_util:get_msg_id()),
MStats1 = MStats0#{nonce => Nonce},
couch_log:report("mango-stats", MStats1).
+
+-spec shard_init() -> any().
+shard_init() ->
+ InitialState = #{docs_examined => 0, keys_examined => 0},
+ put(?SHARD_STATS_KEY, InitialState).
+
+-spec shard_incr_keys_examined() -> any().
+shard_incr_keys_examined() ->
+ incr(keys_examined).
+
+-spec shard_incr_docs_examined() -> any().
+shard_incr_docs_examined() ->
+ incr(docs_examined).
+
+-spec incr(atom()) -> any().
+incr(Key) ->
+ case get(?SHARD_STATS_KEY) of
+ #{} = Stats0 ->
+ Stats = maps:update_with(Key, fun(X) -> X + 1 end, Stats0),
+ put(?SHARD_STATS_KEY, Stats);
+ _ ->
+ ok
+ end.
+
+-spec shard_get_stats() -> shard_stats_v2().
+shard_get_stats() ->
+ get(?SHARD_STATS_KEY).
diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py
index 537a19add..a8f996136 100644
--- a/src/mango/test/15-execution-stats-test.py
+++ b/src/mango/test/15-execution-stats-test.py
@@ -20,7 +20,7 @@ class ExecutionStatsTests(mango.UserDocsTests):
def test_simple_json_index(self):
resp = self.db.find({"age": {"$lt": 35}}, return_raw=True, executionStats=True)
self.assertEqual(len(resp["docs"]), 3)
- self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["total_keys_examined"], 3)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
@@ -38,7 +38,7 @@ class ExecutionStatsTests(mango.UserDocsTests):
{"age": {"$lt": 35}}, return_raw=True, r=3, executionStats=True
)
self.assertEqual(len(resp["docs"]), 3)
- self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["total_keys_examined"], 3)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 3)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
@@ -60,6 +60,19 @@ class ExecutionStatsTests(mango.UserDocsTests):
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
self.assertEqual(resp["execution_stats"]["results_returned"], 0)
+ def test_covering_json_index(self):
+ resp = self.db.find(
+ {"age": {"$lt": 35}},
+ fields=["_id", "age"],
+ return_raw=True,
+ executionStats=True,
+ )
+ self.assertEqual(len(resp["docs"]), 3)
+ self.assertEqual(resp["execution_stats"]["total_keys_examined"], 3)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["results_returned"], 3)
+
@unittest.skipUnless(mango.has_text_service(), "requires text service")
class ExecutionStatsTests_Text(mango.UserDocsTextTests):