basic execution statistics for _find (#768)

Accept an "execution_stats" parameter to _find. If present, return
a new "execution_stats" object in the response which contains
information about the query executed. Currently, this is only
implemented for json/all_docs indexes and contains:
 - total keys examined (currently always 0 for json indexes)
 - total documents examined (when include_docs=true used)
 - total quorum documents examined (when fabric doc lookups used)

9 files changed, 230 insertions, 28 deletions

diff --git a/src/mango/src/mango_cursor.hrl b/src/mango/src/mango_cursor.hrl
index 956466c89..e204c1735 100644
--- a/src/mango/src/mango_cursor.hrl
+++ b/src/mango/src/mango_cursor.hrl
@@ -10,6 +10,9 @@
 % License for the specific language governing permissions and limitations under
 % the License.
 
+-include("mango_execution_stats.hrl").
+
+
 -record(cursor, {
     db,
     index,
@@ -21,6 +24,7 @@
     fields = undefined,
     user_fun,
     user_acc,
+    execution_stats = #execution_stats{},
     bookmark,
     bookmark_docid,
     bookmark_key
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index 96e365a49..ea62cd68c 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -38,7 +38,8 @@
     skip,
     user_fun,
     user_acc,
-    fields
+    fields,
+    execution_stats
 }).
 
 
@@ -87,7 +88,8 @@ execute(Cursor, UserFun, UserAcc) ->
         limit = Limit,
         skip = Skip,
         selector = Selector,
-        opts = Opts
+        opts = Opts,
+        execution_stats = Stats
     } = Cursor,
     QueryArgs = #index_query_args{
         q = mango_selector_text:convert(Selector),
@@ -105,7 +107,8 @@ execute(Cursor, UserFun, UserAcc) ->
         query_args = QueryArgs,
         user_fun = UserFun,
         user_acc = UserAcc,
-        fields = Cursor#cursor.fields
+        fields = Cursor#cursor.fields,
+        execution_stats = mango_execution_stats:log_start(Stats)
     },
     try
         execute(CAcc)
@@ -114,12 +117,14 @@ execute(Cursor, UserFun, UserAcc) ->
             #cacc{
                 bookmark = FinalBM,
                 user_fun = UserFun,
-                user_acc = LastUserAcc
+                user_acc = LastUserAcc,
+                execution_stats = Stats0
             } = FinalCAcc,
             JsonBM = dreyfus_bookmark:pack(FinalBM),
             Arg = {add_key, bookmark, JsonBM},
             {_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
-            {ok, FinalUserAcc}
+            FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
+            {ok, FinalUserAcc0}
     end.
 
 
@@ -165,25 +170,28 @@ handle_hits(CAcc0, [{Sort, Doc} | Rest]) ->
 handle_hit(CAcc0, Sort, Doc) ->
     #cacc{
         limit = Limit,
-        skip = Skip
+        skip = Skip,
+        execution_stats = Stats
     } = CAcc0,
     CAcc1 = update_bookmark(CAcc0, Sort),
-    case mango_selector:match(CAcc1#cacc.selector, Doc) of
+    Stats1 = mango_execution_stats:incr_docs_examined(Stats),
+    CAcc2 = CAcc1#cacc{execution_stats = Stats1},
+    case mango_selector:match(CAcc2#cacc.selector, Doc) of
         true when Skip > 0 ->
-            CAcc1#cacc{skip = Skip - 1};
+            CAcc2#cacc{skip = Skip - 1};
         true when Limit == 0 ->
             % We hit this case if the user spcified with a
             % zero limit. Notice that in this case we need
             % to return the bookmark from before this match
             throw({stop, CAcc0});
         true when Limit == 1 ->
-            NewCAcc = apply_user_fun(CAcc1, Doc),
+            NewCAcc = apply_user_fun(CAcc2, Doc),
             throw({stop, NewCAcc});
         true when Limit > 1 ->
-            NewCAcc = apply_user_fun(CAcc1, Doc),
+            NewCAcc = apply_user_fun(CAcc2, Doc),
             NewCAcc#cacc{limit = Limit - 1};
         false ->
-            CAcc1
+            CAcc2
     end.
 
 
@@ -191,13 +199,15 @@ apply_user_fun(CAcc, Doc) ->
     FinalDoc = mango_fields:extract(Doc, CAcc#cacc.fields),
     #cacc{
         user_fun = UserFun,
-        user_acc = UserAcc
+        user_acc = UserAcc,
+        execution_stats = Stats
     } = CAcc,
+    Stats0 = mango_execution_stats:incr_results_returned(Stats),
     case UserFun({row, FinalDoc}, UserAcc) of
         {ok, NewUserAcc} ->
-            CAcc#cacc{user_acc = NewUserAcc};
+            CAcc#cacc{user_acc = NewUserAcc, execution_stats = Stats0};
         {stop, NewUserAcc} ->
-            throw({stop, CAcc#cacc{user_acc = NewUserAcc}})
+            throw({stop, CAcc#cacc{user_acc = NewUserAcc, execution_stats = Stats0}})
     end.
 
 
@@ -296,6 +306,7 @@ get_json_docs(DbName, Hits) ->
     Ids = lists:map(fun(#sortable{item = Item}) ->
         couch_util:get_value(<<"_id">>, Item#hit.fields)
     end, Hits),
+    % TODO: respect R query parameter (same as json indexes)
     {ok, IdDocs} = dreyfus_fabric:get_json_docs(DbName, Ids),
     lists:map(fun(#sortable{item = Item} = Sort) ->
         Id = couch_util:get_value(<<"_id">>, Item#hit.fields),
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 189ec6674..2dcf1c71b 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -70,10 +70,11 @@ explain(Cursor) ->
     end.
 
 
-execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) ->
+execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
     Cursor = Cursor0#cursor{
         user_fun = UserFun,
-        user_acc = UserAcc
+        user_acc = UserAcc,
+        execution_stats = mango_execution_stats:log_start(Stats)
     },
     case Cursor#cursor.ranges of
         [empty] ->
@@ -108,7 +109,9 @@ execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) ->
                     NewBookmark = mango_json_bookmark:create(LastCursor),
                     Arg = {add_key, bookmark, NewBookmark},
                     {_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc),
-                    {ok, FinalUserAcc};
+                    Stats0 = LastCursor#cursor.execution_stats,
+                    FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
+                    {ok, FinalUserAcc0};
                 {error, Reason} ->
                     {error, Reason}
             end
@@ -182,15 +185,21 @@ choose_best_index(_DbName, IndexRanges) ->
 handle_message({meta, _}, Cursor) ->
     {ok, Cursor};
 handle_message({row, Props}, Cursor) ->
-    case doc_member(Cursor#cursor.db, Props, Cursor#cursor.opts) of
-        {ok, Doc} ->
-            case mango_selector:match(Cursor#cursor.selector, Doc) of
+    case doc_member(Cursor#cursor.db, Props, Cursor#cursor.opts, Cursor#cursor.execution_stats) of
+        {ok, Doc, {execution_stats, ExecutionStats1}} ->
+            Cursor1 = Cursor#cursor {
+                execution_stats = ExecutionStats1
+            },
+            case mango_selector:match(Cursor1#cursor.selector, Doc) of
                 true ->
-                    Cursor1 = update_bookmark_keys(Cursor, Props),
-                    FinalDoc = mango_fields:extract(Doc, Cursor1#cursor.fields),
-                    handle_doc(Cursor1, FinalDoc);
+                    Cursor2 = update_bookmark_keys(Cursor1, Props),
+                    FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields),
+                    Cursor3 = Cursor2#cursor {
+                        execution_stats = mango_execution_stats:incr_results_returned(Cursor2#cursor.execution_stats)
+                    },
+                    handle_doc(Cursor3, FinalDoc);
                 false ->
-                    {ok, Cursor}
+                    {ok, Cursor1}
             end;
         Error ->
             couch_log:error("~s :: Error loading doc: ~p", [?MODULE, Error]),
@@ -298,15 +307,17 @@ apply_opts([{_, _} | Rest], Args) ->
     apply_opts(Rest, Args).
 
 
-doc_member(Db, RowProps, Opts) ->
+doc_member(Db, RowProps, Opts, ExecutionStats) ->
     case couch_util:get_value(doc, RowProps) of
         {DocProps} ->
-            {ok, {DocProps}};
+            ExecutionStats1 = mango_execution_stats:incr_docs_examined(ExecutionStats),
+            {ok, {DocProps}, {execution_stats, ExecutionStats1}};
         undefined ->
+            ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats),
             Id = couch_util:get_value(id, RowProps),
             case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
                 {ok, #doc{}=Doc} ->
-                    {ok, couch_doc:to_json_obj(Doc, [])};
+                    {ok, couch_doc:to_json_obj(Doc, []), {execution_stats, ExecutionStats1}};
                 Else ->
                     Else
             end
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
new file mode 100644
index 000000000..95b9038a8
--- /dev/null
+++ b/src/mango/src/mango_execution_stats.erl
@@ -0,0 +1,89 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_execution_stats).
+
+
+-export([
+    to_json/1,
+    incr_keys_examined/1,
+    incr_docs_examined/1,
+    incr_quorum_docs_examined/1,
+    incr_results_returned/1,
+    log_start/1,
+    log_end/1,
+    maybe_add_stats/4
+]).
+
+
+-include("mango_cursor.hrl").
+
+
+to_json(Stats) ->
+    {[
+        {total_keys_examined, Stats#execution_stats.totalKeysExamined},
+        {total_docs_examined, Stats#execution_stats.totalDocsExamined},
+        {total_quorum_docs_examined, Stats#execution_stats.totalQuorumDocsExamined},
+        {results_returned, Stats#execution_stats.resultsReturned},
+        {execution_time_ms, Stats#execution_stats.executionTimeMs}
+    ]}.
+
+
+incr_keys_examined(Stats) ->
+    Stats#execution_stats {
+        totalKeysExamined = Stats#execution_stats.totalKeysExamined + 1
+    }.
+
+
+incr_docs_examined(Stats) ->
+    Stats#execution_stats {
+        totalDocsExamined = Stats#execution_stats.totalDocsExamined + 1
+    }.
+
+
+incr_quorum_docs_examined(Stats) ->
+    Stats#execution_stats {
+        totalQuorumDocsExamined = Stats#execution_stats.totalQuorumDocsExamined + 1
+    }.
+
+
+incr_results_returned(Stats) ->
+    Stats#execution_stats {
+        resultsReturned = Stats#execution_stats.resultsReturned + 1
+    }.
+
+
+log_start(Stats) ->
+    Stats#execution_stats {
+        executionStartTime = now()
+    }.
+
+
+log_end(Stats) ->
+    End = now(),
+    Diff = timer:now_diff(End, Stats#execution_stats.executionStartTime) / 1000,
+    Stats#execution_stats {
+        executionTimeMs = Diff
+    }.
+
+
+maybe_add_stats(Opts, UserFun, Stats, UserAcc) ->
+    case couch_util:get_value(execution_stats, Opts) of
+        true ->
+            Stats0 = log_end(Stats),
+            JSONValue = to_json(Stats0),
+            Arg = {add_key, execution_stats, JSONValue},
+            {_Go, FinalUserAcc} = UserFun(Arg, UserAcc),
+            FinalUserAcc;
+        _ ->
+            UserAcc
+    end.
\ No newline at end of file
diff --git a/src/mango/src/mango_execution_stats.hrl b/src/mango/src/mango_execution_stats.hrl
new file mode 100644
index 000000000..ea5ed5ee8
--- /dev/null
+++ b/src/mango/src/mango_execution_stats.hrl
@@ -0,0 +1,20 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-record(execution_stats, {
+    totalKeysExamined = 0,
+    totalDocsExamined = 0,
+    totalQuorumDocsExamined = 0,
+    resultsReturned = 0,
+    executionStartTime,
+    executionTimeMs
+}).
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 5bc61f57a..a99b0544d 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -21,6 +21,7 @@
 -include_lib("couch/include/couch_db.hrl").
 -include("mango.hrl").
 -include("mango_idx.hrl").
+-include("mango_execution_stats.hrl").
 
 -record(vacc, {
     resp,
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
index fe28d8d94..7bae9c90d 100644
--- a/src/mango/src/mango_opts.erl
+++ b/src/mango/src/mango_opts.erl
@@ -146,6 +146,12 @@ validate_find({Props}) ->
             {optional, true},
             {default, false},
             {validator, fun mango_opts:is_boolean/1}
+        ]},
+        {<<"execution_stats">>, [
+            {tag, execution_stats},
+            {optional, true},
+            {default, false},
+            {validator, fun mango_opts:is_boolean/1}
         ]}
     ],
     validate(Props, Opts).
diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py
new file mode 100644
index 000000000..67c9e64ec
--- /dev/null
+++ b/src/mango/test/15-execution-stats-test.py
@@ -0,0 +1,58 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+
+import mango
+import unittest
+
+class ExecutionStatsTests(mango.UserDocsTests):
+
+    def test_simple_json_index(self):
+        resp = self.db.find({"age": {"$lt": 35}}, return_raw=True, executionStats=True)
+        self.assertEqual(len(resp["docs"]), 3)
+        self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
+        self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
+        self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
+        self.assertEqual(resp["execution_stats"]["results_returned"], 3)
+        self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
+
+    def test_no_execution_stats(self):
+        resp = self.db.find({"age": {"$lt": 35}}, return_raw=True, executionStats=False)
+        assert "execution_stats" not in resp
+
+    def test_quorum_json_index(self):
+        resp = self.db.find({"age": {"$lt": 35}}, return_raw=True, r=3, executionStats=True)
+        self.assertEqual(len(resp["docs"]), 3)
+        self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
+        self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0)
+        self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 3)
+        self.assertEqual(resp["execution_stats"]["results_returned"], 3)
+        self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class ExecutionStatsTests_Text(mango.UserDocsTextTests):
+
+    def test_simple_text_index(self):
+        resp = self.db.find({"$text": "Stephanie"}, 
+                return_raw=True, 
+                executionStats=True)
+        self.assertEqual(len(resp["docs"]), 1)
+        self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
+        self.assertEqual(resp["execution_stats"]["total_docs_examined"], 1)
+        self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
+        self.assertEqual(resp["execution_stats"]["results_returned"], 1)
+        self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
+
+    def test_no_execution_stats(self):
+        resp = self.db.find({"$text": "Stephanie"}, 
+                return_raw=True)
+        self.assertNotIn("execution_stats", resp)
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index efcf67f2c..dbe980e29 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -155,7 +155,7 @@ class Database(object):
 
     def find(self, selector, limit=25, skip=0, sort=None, fields=None,
                 r=1, conflicts=False, use_index=None, explain=False,
-                bookmark=None, return_raw=False, update=True):
+                bookmark=None, return_raw=False, update=True, executionStats=False):
         body = {
             "selector": selector,
             "use_index": use_index,
@@ -172,6 +172,8 @@ class Database(object):
             body["bookmark"] = bookmark
         if update == False:
             body["update"] = False
+        if executionStats == True:
+            body["execution_stats"] = True
         body = json.dumps(body)
         if explain:
             path = self.path("_explain")