diff options
author | Will Holley <willholley@gmail.com> | 2017-08-30 10:51:06 +0100 |
---|---|---|
committer | garren smith <garren.smith@gmail.com> | 2017-08-30 11:51:06 +0200 |
commit | af839e101a25cb8abc98b2571ef5344a47df470a (patch) | |
tree | ea6875054ad4bdd731b01036883b8584f8f248e3 | |
parent | 64a48bad6203c6bbc646c082abb3d878202cce10 (diff) | |
download | couchdb-af839e101a25cb8abc98b2571ef5344a47df470a.tar.gz |
basic execution statistics for _find (#768)
Accept an "execution_stats" parameter to _find. If present, return
a new "execution_stats" object in the response which contains
information about the query executed. Currently, this is only
implemented for json/all_docs indexes and contains:
- total keys examined (currently always 0 for json indexes)
- total documents examined (when include_docs=true used)
- total quorum documents examined (when fabric doc lookups used)
-rw-r--r-- | src/mango/src/mango_cursor.hrl | 4 | ||||
-rw-r--r-- | src/mango/src/mango_cursor_text.erl | 39 | ||||
-rw-r--r-- | src/mango/src/mango_cursor_view.erl | 37 | ||||
-rw-r--r-- | src/mango/src/mango_execution_stats.erl | 89 | ||||
-rw-r--r-- | src/mango/src/mango_execution_stats.hrl | 20 | ||||
-rw-r--r-- | src/mango/src/mango_httpd.erl | 1 | ||||
-rw-r--r-- | src/mango/src/mango_opts.erl | 6 | ||||
-rw-r--r-- | src/mango/test/15-execution-stats-test.py | 58 | ||||
-rw-r--r-- | src/mango/test/mango.py | 4 |
9 files changed, 230 insertions, 28 deletions
diff --git a/src/mango/src/mango_cursor.hrl b/src/mango/src/mango_cursor.hrl index 956466c89..e204c1735 100644 --- a/src/mango/src/mango_cursor.hrl +++ b/src/mango/src/mango_cursor.hrl @@ -10,6 +10,9 @@ % License for the specific language governing permissions and limitations under % the License. +-include("mango_execution_stats.hrl"). + + -record(cursor, { db, index, @@ -21,6 +24,7 @@ fields = undefined, user_fun, user_acc, + execution_stats = #execution_stats{}, bookmark, bookmark_docid, bookmark_key diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl index 96e365a49..ea62cd68c 100644 --- a/src/mango/src/mango_cursor_text.erl +++ b/src/mango/src/mango_cursor_text.erl @@ -38,7 +38,8 @@ skip, user_fun, user_acc, - fields + fields, + execution_stats }). @@ -87,7 +88,8 @@ execute(Cursor, UserFun, UserAcc) -> limit = Limit, skip = Skip, selector = Selector, - opts = Opts + opts = Opts, + execution_stats = Stats } = Cursor, QueryArgs = #index_query_args{ q = mango_selector_text:convert(Selector), @@ -105,7 +107,8 @@ execute(Cursor, UserFun, UserAcc) -> query_args = QueryArgs, user_fun = UserFun, user_acc = UserAcc, - fields = Cursor#cursor.fields + fields = Cursor#cursor.fields, + execution_stats = mango_execution_stats:log_start(Stats) }, try execute(CAcc) @@ -114,12 +117,14 @@ execute(Cursor, UserFun, UserAcc) -> #cacc{ bookmark = FinalBM, user_fun = UserFun, - user_acc = LastUserAcc + user_acc = LastUserAcc, + execution_stats = Stats0 } = FinalCAcc, JsonBM = dreyfus_bookmark:pack(FinalBM), Arg = {add_key, bookmark, JsonBM}, {_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc), - {ok, FinalUserAcc} + FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc), + {ok, FinalUserAcc0} end. @@ -165,25 +170,28 @@ handle_hits(CAcc0, [{Sort, Doc} | Rest]) -> handle_hit(CAcc0, Sort, Doc) -> #cacc{ limit = Limit, - skip = Skip + skip = Skip, + execution_stats = Stats } = CAcc0, CAcc1 = update_bookmark(CAcc0, Sort), - case mango_selector:match(CAcc1#cacc.selector, Doc) of + Stats1 = mango_execution_stats:incr_docs_examined(Stats), + CAcc2 = CAcc1#cacc{execution_stats = Stats1}, + case mango_selector:match(CAcc2#cacc.selector, Doc) of true when Skip > 0 -> - CAcc1#cacc{skip = Skip - 1}; + CAcc2#cacc{skip = Skip - 1}; true when Limit == 0 -> % We hit this case if the user spcified with a % zero limit. Notice that in this case we need % to return the bookmark from before this match throw({stop, CAcc0}); true when Limit == 1 -> - NewCAcc = apply_user_fun(CAcc1, Doc), + NewCAcc = apply_user_fun(CAcc2, Doc), throw({stop, NewCAcc}); true when Limit > 1 -> - NewCAcc = apply_user_fun(CAcc1, Doc), + NewCAcc = apply_user_fun(CAcc2, Doc), NewCAcc#cacc{limit = Limit - 1}; false -> - CAcc1 + CAcc2 end. @@ -191,13 +199,15 @@ apply_user_fun(CAcc, Doc) -> FinalDoc = mango_fields:extract(Doc, CAcc#cacc.fields), #cacc{ user_fun = UserFun, - user_acc = UserAcc + user_acc = UserAcc, + execution_stats = Stats } = CAcc, + Stats0 = mango_execution_stats:incr_results_returned(Stats), case UserFun({row, FinalDoc}, UserAcc) of {ok, NewUserAcc} -> - CAcc#cacc{user_acc = NewUserAcc}; + CAcc#cacc{user_acc = NewUserAcc, execution_stats = Stats0}; {stop, NewUserAcc} -> - throw({stop, CAcc#cacc{user_acc = NewUserAcc}}) + throw({stop, CAcc#cacc{user_acc = NewUserAcc, execution_stats = Stats0}}) end. @@ -296,6 +306,7 @@ get_json_docs(DbName, Hits) -> Ids = lists:map(fun(#sortable{item = Item}) -> couch_util:get_value(<<"_id">>, Item#hit.fields) end, Hits), + % TODO: respect R query parameter (same as json indexes) {ok, IdDocs} = dreyfus_fabric:get_json_docs(DbName, Ids), lists:map(fun(#sortable{item = Item} = Sort) -> Id = couch_util:get_value(<<"_id">>, Item#hit.fields), diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index 189ec6674..2dcf1c71b 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -70,10 +70,11 @@ explain(Cursor) -> end. -execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) -> +execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) -> Cursor = Cursor0#cursor{ user_fun = UserFun, - user_acc = UserAcc + user_acc = UserAcc, + execution_stats = mango_execution_stats:log_start(Stats) }, case Cursor#cursor.ranges of [empty] -> @@ -108,7 +109,9 @@ execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) -> NewBookmark = mango_json_bookmark:create(LastCursor), Arg = {add_key, bookmark, NewBookmark}, {_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc), - {ok, FinalUserAcc}; + Stats0 = LastCursor#cursor.execution_stats, + FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc), + {ok, FinalUserAcc0}; {error, Reason} -> {error, Reason} end @@ -182,15 +185,21 @@ choose_best_index(_DbName, IndexRanges) -> handle_message({meta, _}, Cursor) -> {ok, Cursor}; handle_message({row, Props}, Cursor) -> - case doc_member(Cursor#cursor.db, Props, Cursor#cursor.opts) of - {ok, Doc} -> - case mango_selector:match(Cursor#cursor.selector, Doc) of + case doc_member(Cursor#cursor.db, Props, Cursor#cursor.opts, Cursor#cursor.execution_stats) of + {ok, Doc, {execution_stats, ExecutionStats1}} -> + Cursor1 = Cursor#cursor { + execution_stats = ExecutionStats1 + }, + case mango_selector:match(Cursor1#cursor.selector, Doc) of true -> - Cursor1 = update_bookmark_keys(Cursor, Props), - FinalDoc = mango_fields:extract(Doc, Cursor1#cursor.fields), - handle_doc(Cursor1, FinalDoc); + Cursor2 = update_bookmark_keys(Cursor1, Props), + FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields), + Cursor3 = Cursor2#cursor { + execution_stats = mango_execution_stats:incr_results_returned(Cursor2#cursor.execution_stats) + }, + handle_doc(Cursor3, FinalDoc); false -> - {ok, Cursor} + {ok, Cursor1} end; Error -> couch_log:error("~s :: Error loading doc: ~p", [?MODULE, Error]), @@ -298,15 +307,17 @@ apply_opts([{_, _} | Rest], Args) -> apply_opts(Rest, Args). -doc_member(Db, RowProps, Opts) -> +doc_member(Db, RowProps, Opts, ExecutionStats) -> case couch_util:get_value(doc, RowProps) of {DocProps} -> - {ok, {DocProps}}; + ExecutionStats1 = mango_execution_stats:incr_docs_examined(ExecutionStats), + {ok, {DocProps}, {execution_stats, ExecutionStats1}}; undefined -> + ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats), Id = couch_util:get_value(id, RowProps), case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of {ok, #doc{}=Doc} -> - {ok, couch_doc:to_json_obj(Doc, [])}; + {ok, couch_doc:to_json_obj(Doc, []), {execution_stats, ExecutionStats1}}; Else -> Else end diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl new file mode 100644 index 000000000..95b9038a8 --- /dev/null +++ b/src/mango/src/mango_execution_stats.erl @@ -0,0 +1,89 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(mango_execution_stats). + + +-export([ + to_json/1, + incr_keys_examined/1, + incr_docs_examined/1, + incr_quorum_docs_examined/1, + incr_results_returned/1, + log_start/1, + log_end/1, + maybe_add_stats/4 +]). + + +-include("mango_cursor.hrl"). + + +to_json(Stats) -> + {[ + {total_keys_examined, Stats#execution_stats.totalKeysExamined}, + {total_docs_examined, Stats#execution_stats.totalDocsExamined}, + {total_quorum_docs_examined, Stats#execution_stats.totalQuorumDocsExamined}, + {results_returned, Stats#execution_stats.resultsReturned}, + {execution_time_ms, Stats#execution_stats.executionTimeMs} + ]}. + + +incr_keys_examined(Stats) -> + Stats#execution_stats { + totalKeysExamined = Stats#execution_stats.totalKeysExamined + 1 + }. + + +incr_docs_examined(Stats) -> + Stats#execution_stats { + totalDocsExamined = Stats#execution_stats.totalDocsExamined + 1 + }. + + +incr_quorum_docs_examined(Stats) -> + Stats#execution_stats { + totalQuorumDocsExamined = Stats#execution_stats.totalQuorumDocsExamined + 1 + }. + + +incr_results_returned(Stats) -> + Stats#execution_stats { + resultsReturned = Stats#execution_stats.resultsReturned + 1 + }. + + +log_start(Stats) -> + Stats#execution_stats { + executionStartTime = now() + }. + + +log_end(Stats) -> + End = now(), + Diff = timer:now_diff(End, Stats#execution_stats.executionStartTime) / 1000, + Stats#execution_stats { + executionTimeMs = Diff + }. + + +maybe_add_stats(Opts, UserFun, Stats, UserAcc) -> + case couch_util:get_value(execution_stats, Opts) of + true -> + Stats0 = log_end(Stats), + JSONValue = to_json(Stats0), + Arg = {add_key, execution_stats, JSONValue}, + {_Go, FinalUserAcc} = UserFun(Arg, UserAcc), + FinalUserAcc; + _ -> + UserAcc + end.
\ No newline at end of file diff --git a/src/mango/src/mango_execution_stats.hrl b/src/mango/src/mango_execution_stats.hrl new file mode 100644 index 000000000..ea5ed5ee8 --- /dev/null +++ b/src/mango/src/mango_execution_stats.hrl @@ -0,0 +1,20 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-record(execution_stats, { + totalKeysExamined = 0, + totalDocsExamined = 0, + totalQuorumDocsExamined = 0, + resultsReturned = 0, + executionStartTime, + executionTimeMs +}). diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl index 5bc61f57a..a99b0544d 100644 --- a/src/mango/src/mango_httpd.erl +++ b/src/mango/src/mango_httpd.erl @@ -21,6 +21,7 @@ -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). -include("mango_idx.hrl"). +-include("mango_execution_stats.hrl"). -record(vacc, { resp, diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl index fe28d8d94..7bae9c90d 100644 --- a/src/mango/src/mango_opts.erl +++ b/src/mango/src/mango_opts.erl @@ -146,6 +146,12 @@ validate_find({Props}) -> {optional, true}, {default, false}, {validator, fun mango_opts:is_boolean/1} + ]}, + {<<"execution_stats">>, [ + {tag, execution_stats}, + {optional, true}, + {default, false}, + {validator, fun mango_opts:is_boolean/1} ]} ], validate(Props, Opts). diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py new file mode 100644 index 000000000..67c9e64ec --- /dev/null +++ b/src/mango/test/15-execution-stats-test.py @@ -0,0 +1,58 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + + +import mango +import unittest + +class ExecutionStatsTests(mango.UserDocsTests): + + def test_simple_json_index(self): + resp = self.db.find({"age": {"$lt": 35}}, return_raw=True, executionStats=True) + self.assertEqual(len(resp["docs"]), 3) + self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0) + self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3) + self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0) + self.assertEqual(resp["execution_stats"]["results_returned"], 3) + self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0) + + def test_no_execution_stats(self): + resp = self.db.find({"age": {"$lt": 35}}, return_raw=True, executionStats=False) + assert "execution_stats" not in resp + + def test_quorum_json_index(self): + resp = self.db.find({"age": {"$lt": 35}}, return_raw=True, r=3, executionStats=True) + self.assertEqual(len(resp["docs"]), 3) + self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0) + self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0) + self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 3) + self.assertEqual(resp["execution_stats"]["results_returned"], 3) + self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0) + +@unittest.skipUnless(mango.has_text_service(), "requires text service") +class ExecutionStatsTests_Text(mango.UserDocsTextTests): + + def test_simple_text_index(self): + resp = self.db.find({"$text": "Stephanie"}, + return_raw=True, + executionStats=True) + self.assertEqual(len(resp["docs"]), 1) + self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0) + self.assertEqual(resp["execution_stats"]["total_docs_examined"], 1) + self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0) + self.assertEqual(resp["execution_stats"]["results_returned"], 1) + self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0) + + def test_no_execution_stats(self): + resp = self.db.find({"$text": "Stephanie"}, + return_raw=True) + self.assertNotIn("execution_stats", resp) diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py index efcf67f2c..dbe980e29 100644 --- a/src/mango/test/mango.py +++ b/src/mango/test/mango.py @@ -155,7 +155,7 @@ class Database(object): def find(self, selector, limit=25, skip=0, sort=None, fields=None, r=1, conflicts=False, use_index=None, explain=False, - bookmark=None, return_raw=False, update=True): + bookmark=None, return_raw=False, update=True, executionStats=False): body = { "selector": selector, "use_index": use_index, @@ -172,6 +172,8 @@ class Database(object): body["bookmark"] = bookmark if update == False: body["update"] = False + if executionStats == True: + body["execution_stats"] = True body = json.dumps(body) if explain: path = self.path("_explain") |