add bookmark support

4 files changed, 70 insertions, 39 deletions

diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 15eb55d92..54107ec79 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -99,6 +99,9 @@ maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
 maybe_replace_max_json(EndKey) ->
     EndKey.
 
+%% TODO: When supported, handle:
+%% partitions
+%% conflicts
 index_args(#cursor{} = Cursor) ->
     #cursor{
         index = Idx,
@@ -106,6 +109,7 @@ index_args(#cursor{} = Cursor) ->
         bookmark = Bookmark
     } = Cursor,
     io:format("SELE ~p ranges ~p ~n", [Cursor#cursor.selector, Cursor#cursor.ranges]),
+
     Args0 = #{
         start_key => mango_idx:start_key(Idx, Cursor#cursor.ranges),
         start_key_docid => <<>>,
@@ -113,19 +117,27 @@ index_args(#cursor{} = Cursor) ->
         end_key_docid => <<255>>,
         skip => 0
     },
-    Args = mango_json_bookmark:update_args(Bookmark, Args0),
 
     Sort = couch_util:get_value(sort, Opts, [<<"asc">>]),
-    io:format("SORT ~p ~n", [Sort]),
     Args1 = case mango_sort:directions(Sort) of
-        [<<"desc">> | _] -> Args#{dir => rev};
-        _ -> Args#{dir => fwd}
+        [<<"desc">> | _] ->
+            #{
+                start_key := SK,
+                start_key_docid := SKDI,
+                end_key := EK,
+                end_key_docid := EKDI
+            } = Args0,
+            Args0#{
+                dir => rev,
+                start_key => EK,
+                start_key_docid => EKDI,
+                end_key => SK,
+                end_key_docid => SKDI
+            };
+        _ ->
+            Args0#{dir => fwd}
     end,
-
-    %% TODO: When supported, handle:
-    %% partitions
-    %% conflicts
-    Args1.
+    mango_json_bookmark:update_args(Bookmark, Args1).
 
 
 execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
@@ -299,14 +311,14 @@ set_mango_msg_timestamp() ->
 
 handle_message({meta, _}, Cursor) ->
     {ok, Cursor};
-handle_message({doc, Doc}, Cursor) ->
+handle_message({doc, Key, Doc}, Cursor) ->
     case doc_member(Cursor, Doc) of
         {ok, Doc, {execution_stats, ExecutionStats1}} ->
             Cursor1 = Cursor#cursor {
                 execution_stats = ExecutionStats1
             },
             {Props} = Doc,
-            Cursor2 = update_bookmark_keys(Cursor1, Props),
+            Cursor2 = update_bookmark_keys(Cursor1, {Key, Props}),
             FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields),
             handle_doc(Cursor2, FinalDoc);
         {no_match, _, {execution_stats, ExecutionStats1}} ->
@@ -329,8 +341,10 @@ handle_all_docs_message({row, Props}, Cursor) ->
     case is_design_doc(Props) of
         true -> {ok, Cursor};
         false ->
-            {doc, Doc} = lists:keyfind(doc, 1, Props),
-            handle_message({doc, Doc}, Cursor)
+            Doc = couch_util:get_value(doc, Props),
+            Key = couch_util:get_value(key, Props),
+
+            handle_message({doc, Key, Doc}, Cursor)
     end;
 handle_all_docs_message(Message, Cursor) ->
     handle_message(Message, Cursor).
@@ -480,9 +494,11 @@ is_design_doc(RowProps) ->
     end.
 
 
-update_bookmark_keys(#cursor{limit = Limit} = Cursor, Props) when Limit > 0 ->
-    Id = couch_util:get_value(id, Props), 
-    Key = couch_util:get_value(key, Props), 
+update_bookmark_keys(#cursor{limit = Limit} = Cursor, {Key, Props}) when Limit > 0 ->
+    io:format("PROPS ~p ~n", [Props]),
+    Id = couch_util:get_value(<<"_id">>, Props),
+%%    Key = couch_util:get_value(<<"key">>, Props),
+    io:format("BOOMARK KEYS id ~p key ~p ~n", [Id, Key]),
     Cursor#cursor {
         bookmark_docid = Id,
         bookmark_key = Key
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index 3e0c48e74..9a17a85f3 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -121,7 +121,8 @@ write_doc(TxDb, DocId, IdxResults) ->
 
 
 query_all_docs(Db, CallBack, Cursor, Args) ->
-    Opts = args_to_fdb_opts(Args) ++ [{include_docs, true}],
+    Opts = args_to_fdb_opts(Args, true) ++ [{include_docs, true}],
+    io:format("ALL DOC OPTS ~p ~n", [Opts]),
     fabric2_db:fold_docs(Db, CallBack, Cursor, Opts).
 
 
@@ -138,7 +139,7 @@ query(Db, CallBack, Cursor, Args) ->
             callback => CallBack
         },
 
-        Opts = args_to_fdb_opts(Args),
+        Opts = args_to_fdb_opts(Args, false),
         io:format("OPTS ~p ~n", [Opts]),
         try
             Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, Opts),
@@ -153,7 +154,7 @@ query(Db, CallBack, Cursor, Args) ->
     end).
 
 
-args_to_fdb_opts(Args) ->
+args_to_fdb_opts(Args, AllDocs) ->
     #{
         start_key := StartKey0,
         start_key_docid := StartKeyDocId,
@@ -165,16 +166,6 @@ args_to_fdb_opts(Args) ->
 
     io:format("ARGS ~p ~n", [Args]),
     io:format("START ~p ~n End ~p ~n", [StartKey0, EndKey0]),
-%%    StartKey1 = if StartKey0 == undefined -> undefined; true ->
-%%        couch_views_encoding:encode(StartKey0, key)
-%%    end,
-
-    % fabric2_fdb:fold_range switches keys around because map/reduce switches them
-    % but we do need to switch them. So we do this fun dance
-    {StartKeyName, EndKeyName} = case Direction of
-        rev -> {end_key, start_key};
-        _ -> {start_key, end_key}
-    end,
 
     StartKeyOpts = case {StartKey0, StartKeyDocId} of
         {[], _} ->
@@ -182,11 +173,16 @@ args_to_fdb_opts(Args) ->
         {null, _} ->
             %% all_docs no startkey
             [];
-%%        {undefined, _} ->
-%%            [];
+        {StartKey0, _} when AllDocs == true ->
+            StartKey1 = if is_binary(StartKey0) -> StartKey0; true ->
+                %% couch_views_encoding:encode(StartKey0, key)
+                couch_util:to_binary(StartKey0)
+            end,
+            io:format("START SEction ~p ~n", [StartKey1]),
+            [{start_key, StartKey1}];
         {StartKey0, StartKeyDocId} ->
             StartKey1 = couch_views_encoding:encode(StartKey0, key),
-            [{StartKeyName, {StartKey1, StartKeyDocId}}]
+            [{start_key, {StartKey1, StartKeyDocId}}]
     end,
 
     InclusiveEnd = true,
@@ -201,11 +197,18 @@ args_to_fdb_opts(Args) ->
         {[<<255>>], _, _} ->
             %% mango index no endkey with a $lt in selector
             [];
+        {EndKey0, EndKeyDocId, _} when AllDocs == true ->
+            EndKey1 = if is_binary(EndKey0) -> EndKey0; true ->
+                couch_util:to_binary(EndKey0)
+                end,
+            io:format("ENDKEY ~p ~n", [EndKey1]),
+            [{end_key, EndKey1}];
         {EndKey0, EndKeyDocId, _} when InclusiveEnd ->
             EndKey1 = couch_views_encoding:encode(EndKey0, key),
-            [{EndKeyName, {EndKey1, EndKeyDocId}}]
+            [{end_key, {EndKey1, EndKeyDocId}}]
     end,
 
+
     [
         {skip, Skip},
         {dir, Direction},
@@ -213,7 +216,7 @@ args_to_fdb_opts(Args) ->
     ] ++ StartKeyOpts ++ EndKeyOpts.
 
 
-fold_cb({Key, _}, Acc) ->
+fold_cb({Key, Val}, Acc) ->
     #{
         prefix := MangoIdxPrefix,
         db := Db,
@@ -222,10 +225,11 @@ fold_cb({Key, _}, Acc) ->
 
     } = Acc,
     {{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
+    SortKeys = couch_views_encoding:decode(Val),
     {ok, Doc} = fabric2_db:open_doc(Db, DocId),
     JSONDoc = couch_doc:to_json_obj(Doc, []),
     io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
-    case Callback({doc, JSONDoc}, Cursor) of
+    case Callback({doc, SortKeys, JSONDoc}, Cursor) of
         {ok, Cursor1} ->
             Acc#{
                 cursor := Cursor1
@@ -274,5 +278,6 @@ add_key(TxDb, MangoIdxPrefix, Results, DocId) ->
         tx := Tx
     } = TxDb,
     Key = create_key(MangoIdxPrefix, Results, DocId),
-    erlfdb:set(Tx, Key, <<0>>).
+    Val = couch_views_encoding:encode(Results),
+    erlfdb:set(Tx, Key, Val).
 
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index c1deaa971..5be8530d7 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -322,7 +322,6 @@ start_key(#idx{}=Idx, Ranges) ->
 
 end_key(#idx{}=Idx, Ranges) ->
     Mod = idx_mod(Idx),
-    io:format("END KEY ~p ~n", [Mod]),
     Mod:end_key(Ranges).
 
 
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 5b1c7a77c..62d6c1b72 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -138,7 +138,8 @@ class Database(object):
         name=None,
         ddoc=None,
         partial_filter_selector=None,
-        selector=None
+        selector=None,
+        wait_for_built_index=True
     ):
         body = {"index": {"fields": fields}, "type": idx_type, "w": 3}
         if name is not None:
@@ -156,7 +157,7 @@ class Database(object):
         assert r.json()["name"] is not None
 
         created = r.json()["result"] == "created"
-        if created:
+        if created and wait_for_built_index:
             # wait until the database reports the index as available and build
             while len([
                     i
@@ -167,6 +168,16 @@ class Database(object):
 
         return created
 
+    def wait_for_built_indexes(self):
+        indexes = self.list_indexes()
+        while len([
+            i
+            for i in self.list_indexes()
+            if i["build_status"] == "ready"
+        ]) < len(indexes):
+            delay(t=0.2)
+
+
     def create_text_index(
         self,
         analyzer=None,