Add mango indexing

This uses couch_views_updater to create mango indexes in the doc update
along with the couch_views_indexer to update the indexes in the
background up to the creation versionstamp.

7 files changed, 187 insertions, 19 deletions

diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index fd0aa7763..d2a2c7257 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -355,6 +355,7 @@ os_process_limit = 100
 ; beahvior for executing provided code in design
 ; documents.
 javascript = couch_js
+query = mango_eval
 
 [mango]
 ; Set to true to disable the "index all fields" text index, which can lead
diff --git a/src/couch_eval/src/couch_eval.erl b/src/couch_eval/src/couch_eval.erl
index 23ca263ab..3541a5b94 100644
--- a/src/couch_eval/src/couch_eval.erl
+++ b/src/couch_eval/src/couch_eval.erl
@@ -75,6 +75,9 @@ acquire_map_context(DbName, DDocId, Language, Sig, Lib, MapFuns) ->
 
 
 -spec release_map_context(context()) -> ok | {error, any()}.
+release_map_context(nil) ->
+    ok;
+
 release_map_context({ApiMod, Ctx}) ->
     ApiMod:release_map_context(Ctx).
 
diff --git a/src/mango/src/mango_eval.erl b/src/mango/src/mango_eval.erl
new file mode 100644
index 000000000..59d784b49
--- /dev/null
+++ b/src/mango/src/mango_eval.erl
@@ -0,0 +1,115 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_eval).
+-behavior(couch_eval).
+
+
+-export([
+    acquire_map_context/1,
+    release_map_context/1,
+    map_docs/2
+]).
+
+
+-export([
+    index_doc/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango_idx.hrl").
+
+
+acquire_map_context(Opts) ->
+    #{
+        db_name := DbName,
+        ddoc_id := DDocId,
+        map_funs := MapFuns
+    } = Opts,
+    Indexes = lists:map(fun (Def) ->
+        #idx{
+            type = <<"json">>,
+            dbname = DbName,
+            ddoc = DDocId,
+            def = Def
+        }
+    end, MapFuns),
+    {ok, Indexes}.
+
+
+release_map_context(_) ->
+    ok.
+
+
+map_docs(Indexes, Docs) ->
+    {ok, lists:map(fun(Doc) ->
+        Json = couch_doc:to_json_obj(Doc, []),
+        Results = index_doc(Indexes, Json),
+        {Doc#doc.id, Results}
+    end, Docs)}.
+
+
+index_doc(Indexes, Doc) ->
+    lists:map(fun(Idx) ->
+        {IdxDef} = mango_idx:def(Idx),
+        Results = get_index_entries(IdxDef, Doc),
+        case lists:member(not_found, Results) of
+            true ->
+                [];
+            false ->
+                [{Results, null}]
+        end
+    end, Indexes).
+
+
+get_index_entries(IdxDef, Doc) ->
+    {Fields} = couch_util:get_value(<<"fields">>, IdxDef),
+    Selector = get_index_partial_filter_selector(IdxDef),
+    case should_index(Selector, Doc) of
+        false ->
+            [not_found];
+        true ->
+            get_index_values(Fields, Doc)
+    end.
+
+
+get_index_values(Fields, Doc) ->
+    lists:map(fun({Field, _Dir}) ->
+        case mango_doc:get_field(Doc, Field) of
+            not_found -> not_found;
+            bad_path -> not_found;
+            Value -> Value
+        end
+    end, Fields).
+
+
+get_index_partial_filter_selector(IdxDef) ->
+    case couch_util:get_value(<<"partial_filter_selector">>, IdxDef, {[]}) of
+        {[]} ->
+            % this is to support legacy text indexes that had the
+            % partial_filter_selector set as selector
+            couch_util:get_value(<<"selector">>, IdxDef, {[]});
+        Else ->
+            Else
+    end.
+
+
+should_index(Selector, Doc) ->
+    NormSelector = mango_selector:normalize(Selector),
+    Matches = mango_selector:match(NormSelector, Doc),
+    IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of
+        <<"_design/", _/binary>> -> true;
+        _ -> false
+    end,
+    Matches and not IsDesign.
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index a26a6851a..ba9f68fd0 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -50,11 +50,35 @@
 -include_lib("couch/include/couch_db.hrl").
 -include("mango.hrl").
 -include("mango_idx.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
 
 
 list(Db) ->
-    {ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE),
-    Indexes.
+    DDocs = couch_views_ddoc:get_mango_list(Db),
+    DbName = fabric2_db:name(Db),
+    Indexes = lists:foldl(fun(DDoc, Acc) ->
+        {Props} = couch_doc:to_json_obj(DDoc, []),
+
+        case proplists:get_value(<<"language">>, Props) == <<"query">> of
+            true ->
+                {ok, Mrst} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
+
+                IsInteractive = couch_views_ddoc:is_interactive(DDoc),
+                BuildState = couch_views_fdb:get_build_status(Db, Mrst),
+
+                Idxs = lists:map(fun(Idx) ->
+                    Idx#idx{
+                        build_status = BuildState,
+                        interactive = IsInteractive
+                    }
+                end, from_ddoc(Db, DDoc)),
+                Acc ++ Idxs;
+            false ->
+                Acc
+        end
+
+    end, [], DDocs),
+    Indexes ++ special(Db).
 
 
 get_usable_indexes(Db, Selector, Opts) ->
@@ -62,13 +86,14 @@ get_usable_indexes(Db, Selector, Opts) ->
     GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
             ExistingIndexes
         ),
+    BuiltIndexes = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
     UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
-    UsableIndexes = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
+    UsableIndexes0 = lists:usort(BuiltIndexes ++ UserSpecifiedIndex),
 
     SortFields = get_sort_fields(Opts),
     UsableFilter = fun(I) -> is_usable(I, Selector, SortFields) end,
 
-    case lists:filter(UsableFilter, UsableIndexes) of
+    case lists:filter(UsableFilter, UsableIndexes0) of
         [] ->
             mango_sort_error(Db, Opts);
         UsableIndexes ->
@@ -162,16 +187,17 @@ delete(Filt, Db, Indexes, DelOpts) ->
     end.
 
 
-from_ddoc(Db, {Props}) ->
+from_ddoc(Db, #doc{id = DDocId} = DDoc) ->
+    {Props} = couch_doc:to_json_obj(DDoc, []),
     DbName = db_to_name(Db),
-    DDoc = proplists:get_value(<<"_id">>, Props),
+    DDocId = proplists:get_value(<<"_id">>, Props),
 
     case proplists:get_value(<<"language">>, Props) of
         <<"query">> -> ok;
         _ ->
             ?MANGO_ERROR(invalid_query_ddoc_language)
     end,
-    IdxMods = case clouseau_rpc:connected() of
+    IdxMods = case is_text_service_available() of
         true ->
             [mango_idx_view, mango_idx_text];
         false ->
@@ -181,7 +207,7 @@ from_ddoc(Db, {Props}) ->
     lists:map(fun(Idx) ->
         Idx#idx{
             dbname = DbName,
-            ddoc = DDoc
+            ddoc = DDocId
         }
     end, Idxs).
 
@@ -192,7 +218,8 @@ special(Db) ->
         name = <<"_all_docs">>,
         type = <<"special">>,
         def = all_docs,
-        opts = []
+        opts = [],
+        build_status = ?INDEX_READY
     },
     % Add one for _update_seq
     [AllDocs].
@@ -278,7 +305,7 @@ db_to_name(Name) when is_binary(Name) ->
 db_to_name(Name) when is_list(Name) ->
     iolist_to_binary(Name);
 db_to_name(Db) ->
-    couch_db:name(Db).
+    fabric2_db:name(Db).
 
 
 get_idx_def(Opts) ->
@@ -293,7 +320,7 @@ get_idx_def(Opts) ->
 get_idx_type(Opts) ->
     case proplists:get_value(type, Opts) of
         <<"json">> -> <<"json">>;
-        <<"text">> -> case clouseau_rpc:connected() of
+        <<"text">> -> case is_text_service_available() of
             true ->
                 <<"text">>;
             false ->
@@ -306,6 +333,11 @@ get_idx_type(Opts) ->
     end.
 
 
+is_text_service_available() ->
+    erlang:function_exported(clouseau_rpc, connected, 0) andalso
+        clouseau_rpc:connected().
+
+
 get_idx_ddoc(Idx, Opts) ->
     case proplists:get_value(ddoc, Opts) of
         <<"_design/", _Rest/binary>> = Name ->
@@ -377,7 +409,8 @@ index(SelectorName, Selector) ->
            <<"Selected">>,<<"json">>,
            {[{<<"fields">>,{[{<<"location">>,<<"asc">>}]}},
              {SelectorName,{Selector}}]},
-           [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}]
+           [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}],
+           <<"ready">>
     }.
 
 get_partial_filter_all_docs_test() ->
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
index 712031b75..68e5aaaf0 100644
--- a/src/mango/src/mango_idx.hrl
+++ b/src/mango/src/mango_idx.hrl
@@ -16,5 +16,7 @@
     name,
     type,
     def,
-    opts
+    opts,
+    build_status,
+    interactive
 }).
diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl
index ac6efc707..3548372b6 100644
--- a/src/mango/src/mango_idx_special.erl
+++ b/src/mango/src/mango_idx_special.erl
@@ -28,6 +28,7 @@
 
 -include_lib("couch/include/couch_db.hrl").
 -include("mango_idx.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
 
 
 validate(_) ->
@@ -55,7 +56,8 @@ to_json(#idx{def=all_docs}) ->
             {<<"fields">>, [{[
                 {<<"_id">>, <<"asc">>}
             ]}]}
-        ]}}
+        ]}},
+        {build_status, ?INDEX_READY}
     ]}.
 
 
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 2d784b638..84be4180b 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -54,7 +54,16 @@ add(#doc{body={Props0}}=DDoc, Idx) ->
     NewView = make_view(Idx),
     Views2 = lists:keystore(element(1, NewView), 1, Views1, NewView),
     Props1 = lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}),
-    {ok, DDoc#doc{body={Props1}}}.
+
+    {Opts0} = proplists:get_value(<<"options">>, Props1, {[]}),
+    Opts1 = case lists:keymember(<<"interactive">>, 1, Opts0) of
+        true -> Opts0;
+        false -> Opts0 ++ [{<<"interactive">>, true}]
+    end,
+    Props2 = lists:keystore(<<"options">>, 1, Props1, {<<"options">>, {Opts1}}),
+
+    Props3 = [{<<"autoupdate">>, false}],
+    {ok, DDoc#doc{body={Props2 ++ Props3}}}.
 
 
 remove(#doc{body={Props0}}=DDoc, Idx) ->
@@ -68,13 +77,15 @@ remove(#doc{body={Props0}}=DDoc, Idx) ->
     if Views2 /= Views1 -> ok; true ->
         ?MANGO_ERROR({index_not_found, Idx#idx.name})
     end,
-    Props1 = case Views2 of
+    Props3 = case Views2 of
         [] ->
-            lists:keydelete(<<"views">>, 1, Props0);
+            Props1 = lists:keydelete(<<"views">>, 1, Props0),
+            Props2 = lists:keydelete(<<"options">>, 1, Props1),
+            lists:keydelete(<<"autoupdate">>, 1, Props2);
         _ ->
             lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}})
     end,
-    {ok, DDoc#doc{body={Props1}}}.
+    {ok, DDoc#doc{body={Props3}}}.
 
 
 from_ddoc({Props}) ->
@@ -104,7 +115,8 @@ to_json(Idx) ->
         {ddoc, Idx#idx.ddoc},
         {name, Idx#idx.name},
         {type, Idx#idx.type},
-        {def, {def_to_json(Idx#idx.def)}}
+        {def, {def_to_json(Idx#idx.def)}},
+        {build_status, Idx#idx.build_status}
     ]}.