Update Mango query to work with couch_views

9 files changed, 139 insertions, 147 deletions

diff --git a/src/couch_views/test/couch_views_map_test.erl b/src/couch_views/test/couch_views_map_test.erl
index f8ba18319..7d1e94b2c 100644
--- a/src/couch_views/test/couch_views_map_test.erl
+++ b/src/couch_views/test/couch_views_map_test.erl
@@ -14,6 +14,7 @@
 
 -include_lib("couch/include/couch_eunit.hrl").
 -include_lib("couch/include/couch_db.hrl").
+-include("couch_views.hrl").
 
 
 -define(TDEF(A), {atom_to_list(A), fun A/0}).
@@ -56,7 +57,8 @@ map_views_test_() ->
                 ?TDEF(should_map_duplicate_keys),
                 ?TDEF(should_map_with_doc_emit),
                 ?TDEF(should_map_update_is_false),
-                ?TDEF(should_map_update_is_lazy)
+                ?TDEF(should_map_update_is_lazy),
+                ?TDEF(should_map_wait_for_interactive)
                 % fun should_give_ext_size_seq_indexed_test/1
             ]
         }
@@ -419,6 +421,25 @@ should_map_update_is_lazy() ->
     ?assertEqual(Expect, Result2).
 
 
+should_map_wait_for_interactive() ->
+    DbName = ?tempdb(),
+    {ok, Db} = fabric2_db:create(DbName, [{user_ctx, ?ADMIN_USER}]),
+
+    DDoc = create_interactive_ddoc(),
+    Docs = make_docs(101),
+
+    fabric2_db:update_docs(Db, Docs),
+    fabric2_db:update_docs(Db, [DDoc]),
+
+    Result = couch_views:query(Db, DDoc, <<"idx_01">>, fun default_cb/2, [],
+        #{limit => 3}),
+    ?assertEqual({ok, [
+        {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
+        {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
+        {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}
+    ]}, Result).
+
+
 % should_give_ext_size_seq_indexed_test(Db) ->
 %     DDoc = couch_doc:from_json_obj({[
 %         {<<"_id">>, <<"_design/seqdoc">>},
@@ -510,6 +531,24 @@ create_ddoc() ->
         ]}}
     ]}).
 
+create_interactive_ddoc() ->
+    couch_doc:from_json_obj({[
+        {<<"_id">>, <<"_design/ddoc_interactive">>},
+        {<<"language">>, <<"javascript">>},
+        {<<"views">>, {[
+            {<<"idx_01">>, {[
+                {<<"map">>, <<
+                    "function(doc) {"
+                        "if (doc.val) {"
+                            "emit(doc.val, doc.val);"
+                        "}"
+                    "}">>}
+            ]}}
+        ]}},
+        {<<"autoupdate">>, false},
+        {<<"interactive">>, true}
+    ]}).
+
 
 make_docs(Count) ->
     [doc(I) || I <- lists:seq(1, Count)].
diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl
index 42717ffc8..66cef65b3 100644
--- a/src/mango/src/mango_crud.erl
+++ b/src/mango/src/mango_crud.erl
@@ -33,9 +33,8 @@ insert(Db, #doc{}=Doc, Opts) ->
     insert(Db, [Doc], Opts);
 insert(Db, {_}=Doc, Opts) ->
     insert(Db, [Doc], Opts);
-insert(Db, Docs, Opts0) when is_list(Docs) ->
-    Opts1 = maybe_add_user_ctx(Db, Opts0),
-    case fabric:update_docs(Db, Docs, Opts1) of
+insert(Db, Docs, Opts) when is_list(Docs) ->
+    case fabric2_db:update_docs(Db, Docs, Opts) of
         {ok, Results0} ->
             {ok, lists:zipwith(fun result_to_json/2, Docs, Results0)};
         {accepted, Results0} ->
@@ -45,9 +44,8 @@ insert(Db, Docs, Opts0) when is_list(Docs) ->
     end.
 
 
-find(Db, Selector, Callback, UserAcc, Opts0) ->
-    Opts1 = maybe_add_user_ctx(Db, Opts0),
-    {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
+find(Db, Selector, Callback, UserAcc, Opts) ->
+    {ok, Cursor} = mango_cursor:create(Db, Selector, Opts),
     mango_cursor:execute(Cursor, Callback, UserAcc).
 
 
@@ -97,21 +95,11 @@ delete(Db, Selector, Options) ->
     end.
 
 
-explain(Db, Selector, Opts0) ->
-    Opts1 = maybe_add_user_ctx(Db, Opts0),
-    {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
+explain(Db, Selector, Opts) ->
+    {ok, Cursor} = mango_cursor:create(Db, Selector, Opts),
     mango_cursor:explain(Cursor).
 
 
-maybe_add_user_ctx(Db, Opts) ->
-    case lists:keyfind(user_ctx, 1, Opts) of
-        {user_ctx, _} ->
-            Opts;
-        false ->
-            [{user_ctx, couch_db:get_user_ctx(Db)} | Opts]
-    end.
-
-
 result_to_json(#doc{id=Id}, Result) ->
     result_to_json(Id, Result);
 result_to_json({Props}, Result) ->
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index db4e98184..63b449cdc 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -48,7 +48,9 @@
 
 create(Db, Selector0, Opts) ->
     Selector = mango_selector:normalize(Selector0),
-    UsableIndexes = mango_idx:get_usable_indexes(Db, Selector, Opts),
+    UsableIndexes = fabric2_fdb:transactional(Db, fun (TxDb) ->
+        mango_idx:get_usable_indexes(TxDb, Selector, Opts)
+    end),
     case mango_cursor:maybe_filter_indexes_by_ddoc(UsableIndexes, Opts) of
         [] ->
             % use_index doesn't match a valid index - fall back to a valid one
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index b88f6eaee..4960fa126 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -31,9 +31,7 @@
 -include_lib("fabric/include/fabric.hrl").
 
 -include("mango_cursor.hrl").
--include("mango_idx_view.hrl").
 
--define(HEARTBEAT_INTERVAL_IN_USEC, 4000000).
 
 create(Db, Indexes, Selector, Opts) ->
     FieldRanges = mango_idx_view:field_ranges(Selector),
@@ -91,7 +89,8 @@ maybe_replace_max_json(?MAX_STR) ->
     <<"<MAX>">>;
 
 maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
-    H1 = if H == ?MAX_JSON_OBJ -> <<"<MAX>">>;
+    MAX_VAL = couch_views_encoding:max(),
+    H1 = if H == MAX_VAL  -> <<"<MAX>">>;
             true -> H
     end,
     [H1 | maybe_replace_max_json(T)];
@@ -100,7 +99,7 @@ maybe_replace_max_json(EndKey) ->
     EndKey.
 
 
-base_args(#cursor{index = Idx, selector = Selector} = Cursor) ->
+base_args(#cursor{index = Idx} = Cursor) ->
     {StartKey, EndKey} = case Cursor#cursor.ranges of
         [empty] ->
             {null, null};
@@ -132,18 +131,19 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
             #cursor{opts = Opts, bookmark = Bookmark} = Cursor,
             Args0 = apply_opts(Opts, BaseArgs),
             Args = mango_json_bookmark:update_args(Bookmark, Args0),
-            UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
-            DbOpts = [{user_ctx, UserCtx}],
             Result = case mango_idx:def(Idx) of
                 all_docs ->
                     CB = fun ?MODULE:handle_all_docs_message/2,
-                    fabric:all_docs(Db, DbOpts, CB, Cursor, Args);
+                    AllDocOpts = fabric2_util:all_docs_view_opts(Args)
+                        ++ [{restart_tx, true}],
+                    fabric2_db:fold_docs(Db, CB, Cursor, AllDocOpts);
                 _ ->
                     CB = fun ?MODULE:handle_message/2,
                     % Normal view
-                    DDoc = ddocid(Idx),
+                    DDocId = mango_idx:ddoc(Idx),
+                    {ok, DDoc} = fabric2_db:open_doc(Db, DDocId),
                     Name = mango_idx:name(Idx),
-                    fabric:query_view(Db, DbOpts, DDoc, Name, CB, Cursor, Args)
+                    couch_views:query(Db, DDoc, Name, CB, Cursor, Args)
             end,
             case Result of
                 {ok, LastCursor} ->
@@ -227,7 +227,7 @@ choose_best_index(_DbName, IndexRanges) ->
 handle_message({meta, _}, Cursor) ->
     {ok, Cursor};
 handle_message({row, Props}, Cursor) ->
-    case doc_member(Cursor, Props) of
+    case match_doc(Cursor, Props) of
         {ok, Doc, {execution_stats, Stats}} ->
             Cursor1 = Cursor#cursor {
                 execution_stats = Stats
@@ -280,15 +280,6 @@ handle_doc(C, _Doc) ->
     {stop, C}.
 
 
-ddocid(Idx) ->
-    case mango_idx:ddoc(Idx) of
-        <<"_design/", Rest/binary>> ->
-            Rest;
-        Else ->
-            Else
-    end.
-
-
 apply_opts([], Args) ->
     Args;
 apply_opts([{conflicts, true} | Rest], Args) ->
@@ -340,41 +331,18 @@ apply_opts([{_, _} | Rest], Args) ->
     apply_opts(Rest, Args).
 
 
-doc_member(Cursor, RowProps) ->
-    Db = Cursor#cursor.db,
-    Opts = Cursor#cursor.opts,
-    ExecutionStats = Cursor#cursor.execution_stats,
-    Selector = Cursor#cursor.selector,
-    case couch_util:get_value(doc, RowProps) of
-        {DocProps} ->
-            % only matching documents are returned; the selector
-            % is evaluated at the shard level in view_cb({row, Row},
-            {ok, {DocProps}, {execution_stats, ExecutionStats}};
-        undefined ->
-            % an undefined doc was returned, indicating we should
-            % perform a quorum fetch
-            ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats),
-            couch_stats:increment_counter([mango, quorum_docs_examined]),
-            Id = couch_util:get_value(id, RowProps),
-            case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
-                {ok, #doc{}=DocProps} ->
-                    Doc = couch_doc:to_json_obj(DocProps, []),
-                    match_doc(Selector, Doc, ExecutionStats1);
-                Else ->
-                    Else
-            end;
-        _ ->
-            % no doc, no match
-            {no_match, null, {execution_stats, ExecutionStats}}
-    end.
-
-
-match_doc(Selector, Doc, ExecutionStats) ->
+match_doc(Cursor, RowProps) ->
+    #cursor{
+        execution_stats = Stats0,
+        selector = Selector
+    } = Cursor,
+    Stats1 = mango_execution_stats:incr_docs_examined(Stats0, 1),
+    Doc = couch_util:get_value(doc, RowProps),
     case mango_selector:match(Selector, Doc) of
         true ->
-            {ok, Doc, {execution_stats, ExecutionStats}};
+            {ok, Doc, {execution_stats, Stats1}};
         false ->
-            {no_match, Doc, {execution_stats, ExecutionStats}}
+            {no_match, Doc, {execution_stats, Stats1}}
     end.
 
 
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 1054c74bb..94aa866d2 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -32,9 +32,8 @@
     threshold = 1490
 }).
 
-handle_req(#httpd{} = Req, Db0) ->
+handle_req(#httpd{} = Req, Db) ->
     try
-        Db = set_user_ctx(Req, Db0),
         handle_req_int(Req, Db)
     catch
         throw:{mango_error, Module, Reason} ->
@@ -61,7 +60,9 @@ handle_req_int(_, _) ->
 handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
     Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end,
         chttpd:qs(Req)),
-    Idxs = lists:sort(mango_idx:list(Db)),
+    Idxs = fabric2_fdb:transactional(Db, fun(TxDb) ->
+        lists:sort(mango_idx:list(TxDb))
+    end),
     JsonIdxs0 = lists:map(fun mango_idx:to_json/1, Idxs),
     TotalRows = length(JsonIdxs0),
     Limit = case couch_util:get_value(limit, Params, TotalRows) of
@@ -87,25 +88,27 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
     {ok, Idx0} = mango_idx:new(Db, Opts),
     {ok, Idx} = mango_idx:validate_new(Idx0, Db),
     DbOpts = [{user_ctx, Req#httpd.user_ctx}, deleted, ejson_body],
-    {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), DbOpts),
-    Id = Idx#idx.ddoc,
-    Name = Idx#idx.name,
-    Status = case mango_idx:add(DDoc, Idx) of
-        {ok, DDoc} ->
-            <<"exists">>;
-        {ok, NewDDoc} ->
-            case mango_crud:insert(Db, NewDDoc, Opts) of
-                {ok, [{RespProps}]} ->
-                    case lists:keyfind(error, 1, RespProps) of
-                        {error, Reason} ->
-                            ?MANGO_ERROR({error_saving_ddoc, Reason});
-                        _ ->
-                            <<"created">>
-                    end;
-                _ ->
-                    ?MANGO_ERROR(error_saving_ddoc)
-            end
-    end,
+    Id = mango_idx:ddoc(Idx),
+    Name = mango_idx:name(Idx),
+    Status = fabric2_fdb:transactional(Db, fun(TxDb) ->
+        {ok, DDoc} = mango_util:load_ddoc(TxDb, Id, DbOpts),
+        case mango_idx:add(DDoc, Idx) of
+            {ok, DDoc} ->
+                <<"exists">>;
+            {ok, NewDDoc} ->
+                case mango_crud:insert(TxDb, NewDDoc, Opts) of
+                    {ok, [{RespProps}]} ->
+                        case lists:keyfind(error, 1, RespProps) of
+                            {error, Reason} ->
+                                ?MANGO_ERROR({error_saving_ddoc, Reason});
+                            _ ->
+                                <<"created">>
+                        end;
+                    _ ->
+                        ?MANGO_ERROR(error_saving_ddoc)
+                end
+        end
+    end),
 	chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]});
 
 handle_index_req(#httpd{path_parts=[_, _]}=Req, _Db) ->
@@ -118,19 +121,21 @@ handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>,
         <<"_bulk_delete">>]}=Req, Db) ->
     chttpd:validate_ctype(Req, "application/json"),
     {ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)),
-    Idxs = mango_idx:list(Db),
-    DDocs = get_bulk_delete_ddocs(Opts),
-    {Success, Fail} = lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
-        DDocId = convert_to_design_id(DDocId0),
-        Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
-        Id = {<<"id">>, DDocId},
-        case mango_idx:delete(Filt, Db, Idxs, Opts) of
-            {ok, true} ->
-                {[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
-            {error, Error} ->
-                {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]}
-        end
-    end, {[], []}, DDocs),
+    {Success, Fail} = fabric2_fdb:transactional(Db, fun (TxDb) ->
+        Idxs = mango_idx:list(TxDb),
+        DDocs = get_bulk_delete_ddocs(Opts),
+        lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
+            DDocId = convert_to_design_id(DDocId0),
+            Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
+            Id = {<<"id">>, DDocId},
+            case mango_idx:delete(Filt, TxDb, Idxs, Opts) of
+                {ok, true} ->
+                    {[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
+                {error, Error} ->
+                    {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]}
+            end
+        end, {[], []}, DDocs)
+    end),
     chttpd:send_json(Req, {[{<<"success">>, Success}, {<<"fail">>, Fail}]});
 
 handle_index_req(#httpd{path_parts=[_, <<"_index">>,
@@ -144,15 +149,18 @@ handle_index_req(#httpd{method='DELETE',
 
 handle_index_req(#httpd{method='DELETE',
         path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) ->
-    Idxs = mango_idx:list(Db),
-    DDocId = convert_to_design_id(DDocId0),
-    Filt = fun(Idx) ->
-        IsDDoc = mango_idx:ddoc(Idx) == DDocId,
-        IsType = mango_idx:type(Idx) == Type,
-        IsName = mango_idx:name(Idx) == Name,
-        IsDDoc andalso IsType andalso IsName
-    end,
-    case mango_idx:delete(Filt, Db, Idxs, []) of
+    Result = fabric2_fdb:transactional(Db, fun(TxDb) ->
+        Idxs = mango_idx:list(TxDb),
+        DDocId = convert_to_design_id(DDocId0),
+        Filt = fun(Idx) ->
+            IsDDoc = mango_idx:ddoc(Idx) == DDocId,
+            IsType = mango_idx:type(Idx) == Type,
+            IsName = mango_idx:name(Idx) == Name,
+            IsDDoc andalso IsType andalso IsName
+        end,
+        mango_idx:delete(Filt, TxDb, Idxs, [])
+    end),
+    case Result of
         {ok, true} ->
             chttpd:send_json(Req, {[{ok, true}]});
         {error, not_found} ->
@@ -170,7 +178,9 @@ handle_explain_req(#httpd{method='POST'}=Req, Db) ->
     Body = chttpd:json_body_obj(Req),
     {ok, Opts0} = mango_opts:validate_find(Body),
     {value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
-    Resp = mango_crud:explain(Db, Sel, Opts),
+    Resp = fabric2_fdb:transactional(Db, fun(TxDb) ->
+        mango_crud:explain(TxDb, Sel, Opts)
+    end),
     chttpd:send_json(Req, Resp);
 
 handle_explain_req(Req, _Db) ->
@@ -195,11 +205,6 @@ handle_find_req(Req, _Db) ->
     chttpd:send_method_not_allowed(Req, "POST").
 
 
-set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
-    {ok, NewDb} = couch_db:set_user_ctx(Db, Ctx),
-    NewDb.
-
-
 get_bulk_delete_ddocs(Opts) ->
     case lists:keyfind(docids, 1, Opts) of
         {docids, DDocs} when is_list(DDocs) ->
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index ba9f68fd0..99a35886f 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -86,7 +86,7 @@ get_usable_indexes(Db, Selector, Opts) ->
     GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
             ExistingIndexes
         ),
-    BuiltIndexes = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
+    BuiltIndexes = remove_unbuilt_indexes(GlobalIndexes),
     UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
     UsableIndexes0 = lists:usort(BuiltIndexes ++ UserSpecifiedIndex),
 
@@ -399,6 +399,17 @@ get_legacy_selector(Def) ->
         Selector -> Selector
     end.
 
+% remove any interactive indexes that are not built. If an index is not
+% interactive than we do not remove it as it will be built when queried
+remove_unbuilt_indexes(Indexes) ->
+    lists:filter(fun(Idx) ->
+        case Idx#idx.interactive of
+            true -> Idx#idx.build_status == ?INDEX_READY;
+            _ -> true
+        end
+    end, Indexes).
+
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 84be4180b..f80cc217b 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -34,7 +34,6 @@
 -include_lib("couch/include/couch_db.hrl").
 -include("mango.hrl").
 -include("mango_idx.hrl").
--include("mango_idx_view.hrl").
 
 
 validate_new(#idx{}=Idx, _Db) ->
@@ -183,11 +182,11 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) ->
 
 
 end_key([]) ->
-    [?MAX_JSON_OBJ];
+    [couch_views_encoding:max()];
 end_key([{_, _, '$lt', Key} | Rest]) ->
     case mango_json:special(Key) of
         true ->
-            [?MAX_JSON_OBJ];
+            [couch_views_encoding:max()];
         false ->
             [Key | end_key(Rest)]
     end;
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
deleted file mode 100644
index 0d213e56e..000000000
--- a/src/mango/src/mango_idx_view.hrl
+++ /dev/null
@@ -1,13 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
\ No newline at end of file
diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl
index 0d31f15f9..18a643985 100644
--- a/src/mango/src/mango_util.erl
+++ b/src/mango/src/mango_util.erl
@@ -85,14 +85,7 @@ open_doc(Db, DocId) ->
 
 
 open_doc(Db, DocId, Options) ->
-    case mango_util:defer(fabric, open_doc, [Db, DocId, Options]) of
-        {ok, Doc} ->
-            {ok, Doc};
-        {not_found, _} ->
-            not_found;
-        _ ->
-            ?MANGO_ERROR({error_loading_doc, DocId})
-    end.
+    fabric2_db:open_doc(Db, DocId, Options).
 
 
 open_ddocs(Db) ->
@@ -111,7 +104,7 @@ load_ddoc(Db, DDocId, DbOpts) ->
     case open_doc(Db, DDocId, DbOpts) of
         {ok, Doc} ->
             {ok, check_lang(Doc)};
-        not_found ->
+        {not_found, missing} ->
             Body = {[
                 {<<"language">>, <<"query">>}
             ]},