initial creation of fdb startkey/endkey

9 files changed, 223 insertions, 138 deletions

diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 7b47a40c9..57cab21f5 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -19,7 +19,7 @@
 ]).
 
 -export([
-    view_cb/2,
+%%    view_cb/2,
     handle_message/2,
     handle_all_docs_message/2,
     composite_indexes/2,
@@ -63,22 +63,23 @@ explain(Cursor) ->
     #cursor{
         opts = Opts
     } = Cursor,
-
-    BaseArgs = base_args(Cursor),
-    Args = apply_opts(Opts, BaseArgs),
-
-    [{mrargs, {[
-        {include_docs, Args#mrargs.include_docs},
-        {view_type, Args#mrargs.view_type},
-        {reduce, Args#mrargs.reduce},
-        {partition, couch_mrview_util:get_extra(Args, partition, null)},
-        {start_key, maybe_replace_max_json(Args#mrargs.start_key)},
-        {end_key, maybe_replace_max_json(Args#mrargs.end_key)},
-        {direction, Args#mrargs.direction},
-        {stable, Args#mrargs.stable},
-        {update, Args#mrargs.update},
-        {conflicts, Args#mrargs.conflicts}
-    ]}}].
+    [{marargs, {[]}}].
+
+%%    BaseArgs = base_args(Cursor),
+%%    Args = apply_opts(Opts, BaseArgs),
+%%
+%%    [{mrargs, {[
+%%        {include_docs, Args#mrargs.include_docs},
+%%        {view_type, Args#mrargs.view_type},
+%%        {reduce, Args#mrargs.reduce},
+%%        {partition, couch_mrview_util:get_extra(Args, partition, null)},
+%%        {start_key, maybe_replace_max_json(Args#mrargs.start_key)},
+%%        {end_key, maybe_replace_max_json(Args#mrargs.end_key)},
+%%        {direction, Args#mrargs.direction},
+%%        {stable, Args#mrargs.stable},
+%%        {update, Args#mrargs.update},
+%%        {conflicts, Args#mrargs.conflicts}
+%%    ]}}].
 
 
 % replace internal values that cannot
@@ -99,15 +100,31 @@ maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
 maybe_replace_max_json(EndKey) ->
     EndKey.
 
-base_args(#cursor{index = Idx, selector = Selector} = Cursor) ->
-    #mrargs{
-        view_type = map,
-        reduce = false,
-        start_key = mango_idx:start_key(Idx, Cursor#cursor.ranges),
-        end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges),
-        include_docs = true,
-        extra = [{callback, {?MODULE, view_cb}}, {selector, Selector}]
-    }.
+index_args(#cursor{index = Idx} = Cursor) ->
+    #cursor{
+        index = Idx,
+        opts = Opts,
+        bookmark = Bookmark
+    } = Cursor,
+    Args0 = #{
+        start_key => mango_idx:start_key(Idx, Cursor#cursor.ranges),
+        start_key_docid => <<>>,
+        end_key => mango_idx:end_key(Idx, Cursor#cursor.ranges),
+        end_key_docid => <<>>,
+        skip => 0
+    },
+    Args = mango_json_bookmark:update_args(Bookmark, Args0),
+
+    Sort = couch_util:get_value(sort, Opts, [<<"asc">>]),
+    Args1 = case mango_sort:directions(Sort) of
+        [<<"desc">> | _] -> Args#{direction => rev};
+        _ -> Args#{direction => fwd}
+    end,
+
+    %% TODO: When supported, handle:
+    %% partitions
+    %% conflicts
+    Args1.
 
 
 execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
@@ -121,21 +138,23 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
             % empty indicates unsatisfiable ranges, so don't perform search
             {ok, UserAcc};
         _ ->
-            BaseArgs = base_args(Cursor),
+            Args = index_args(Cursor),
             #cursor{opts = Opts, bookmark = Bookmark} = Cursor,
-            Args0 = apply_opts(Opts, BaseArgs),
-            Args = mango_json_bookmark:update_args(Bookmark, Args0), 
+%%            Args0 = BaseArgs,
+%%            Args0 = apply_opts(Opts, BaseArgs),
+%%            Args = mango_json_bookmark:update_args(Bookmark, Args0),
             UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
             DbOpts = [{user_ctx, UserCtx}],
             Result = case mango_idx:def(Idx) of
                 all_docs ->
                     CB = fun ?MODULE:handle_all_docs_message/2,
-                    fabric:all_docs(Db, DbOpts, CB, Cursor, Args);
+                    ok;
+%%                    fabric:all_docs(Db, DbOpts, CB, Cursor, Args);
                 _ ->
                     CB = fun ?MODULE:handle_message/2,
                     % Normal view
-                    DDoc = ddocid(Idx),
-                    Name = mango_idx:name(Idx),
+%%                    DDoc = ddocid(Idx),
+%%                    Name = mango_idx:name(Idx),
                     mango_fdb:query(Db, CB, Cursor, Args)
             end,
             case Result of
@@ -343,72 +362,72 @@ ddocid(Idx) ->
     end.
 
 
-apply_opts([], Args) ->
-    Args;
-apply_opts([{r, RStr} | Rest], Args) ->
-    IncludeDocs = case list_to_integer(RStr) of
-        1 ->
-            true;
-        R when R > 1 ->
-            % We don't load the doc in the view query because
-            % we have to do a quorum read in the coordinator
-            % so there's no point.
-            false
-    end,
-    NewArgs = Args#mrargs{include_docs = IncludeDocs},
-    apply_opts(Rest, NewArgs);
-apply_opts([{conflicts, true} | Rest], Args) ->
-    NewArgs = Args#mrargs{conflicts = true},
-    apply_opts(Rest, NewArgs);
-apply_opts([{conflicts, false} | Rest], Args) ->
-    % Ignored cause default
-    apply_opts(Rest, Args);
-apply_opts([{sort, Sort} | Rest], Args) ->
-    % We only support single direction sorts
-    % so nothing fancy here.
-    case mango_sort:directions(Sort) of
-        [] ->
-            apply_opts(Rest, Args);
-        [<<"asc">> | _] ->
-            apply_opts(Rest, Args);
-        [<<"desc">> | _] ->
-            SK = Args#mrargs.start_key,
-            SKDI = Args#mrargs.start_key_docid,
-            EK = Args#mrargs.end_key,
-            EKDI = Args#mrargs.end_key_docid,
-            NewArgs = Args#mrargs{
-                direction = rev,
-                start_key = EK,
-                start_key_docid = EKDI,
-                end_key = SK,
-                end_key_docid = SKDI
-            },
-            apply_opts(Rest, NewArgs)
-    end;
-apply_opts([{stale, ok} | Rest], Args) ->
-    NewArgs = Args#mrargs{
-        stable = true,
-        update = false
-    },
-    apply_opts(Rest, NewArgs);
-apply_opts([{stable, true} | Rest], Args) ->
-    NewArgs = Args#mrargs{
-        stable = true
-    },
-    apply_opts(Rest, NewArgs);
-apply_opts([{update, false} | Rest], Args) ->
-    NewArgs = Args#mrargs{
-        update = false
-    },
-    apply_opts(Rest, NewArgs);
-apply_opts([{partition, <<>>} | Rest], Args) ->
-    apply_opts(Rest, Args);
-apply_opts([{partition, Partition} | Rest], Args) when is_binary(Partition) ->
-    NewArgs = couch_mrview_util:set_extra(Args, partition, Partition),
-    apply_opts(Rest, NewArgs);
-apply_opts([{_, _} | Rest], Args) ->
-    % Ignore unknown options
-    apply_opts(Rest, Args).
+%%apply_opts([], Args) ->
+%%    Args;
+%%apply_opts([{r, RStr} | Rest], Args) ->
+%%    IncludeDocs = case list_to_integer(RStr) of
+%%        1 ->
+%%            true;
+%%        R when R > 1 ->
+%%            % We don't load the doc in the view query because
+%%            % we have to do a quorum read in the coordinator
+%%            % so there's no point.
+%%            false
+%%    end,
+%%    NewArgs = Args#mrargs{include_docs = IncludeDocs},
+%%    apply_opts(Rest, NewArgs);
+%%apply_opts([{conflicts, true} | Rest], Args) ->
+%%    NewArgs = Args#mrargs{conflicts = true},
+%%    apply_opts(Rest, NewArgs);
+%%apply_opts([{conflicts, false} | Rest], Args) ->
+%%    % Ignored cause default
+%%    apply_opts(Rest, Args);
+%%apply_opts([{sort, Sort} | Rest], Args) ->
+%%    % We only support single direction sorts
+%%    % so nothing fancy here.
+%%    case mango_sort:directions(Sort) of
+%%        [] ->
+%%            apply_opts(Rest, Args);
+%%        [<<"asc">> | _] ->
+%%            apply_opts(Rest, Args);
+%%        [<<"desc">> | _] ->
+%%            SK = Args#mrargs.start_key,
+%%            SKDI = Args#mrargs.start_key_docid,
+%%            EK = Args#mrargs.end_key,
+%%            EKDI = Args#mrargs.end_key_docid,
+%%            NewArgs = Args#mrargs{
+%%                direction = rev,
+%%                start_key = EK,
+%%                start_key_docid = EKDI,
+%%                end_key = SK,
+%%                end_key_docid = SKDI
+%%            },
+%%            apply_opts(Rest, NewArgs)
+%%    end;
+%%apply_opts([{stale, ok} | Rest], Args) ->
+%%    NewArgs = Args#mrargs{
+%%        stable = true,
+%%        update = false
+%%    },
+%%    apply_opts(Rest, NewArgs);
+%%apply_opts([{stable, true} | Rest], Args) ->
+%%    NewArgs = Args#mrargs{
+%%        stable = true
+%%    },
+%%    apply_opts(Rest, NewArgs);
+%%apply_opts([{update, false} | Rest], Args) ->
+%%    NewArgs = Args#mrargs{
+%%        update = false
+%%    },
+%%    apply_opts(Rest, NewArgs);
+%%apply_opts([{partition, <<>>} | Rest], Args) ->
+%%    apply_opts(Rest, Args);
+%%apply_opts([{partition, Partition} | Rest], Args) when is_binary(Partition) ->
+%%    NewArgs = couch_mrview_util:set_extra(Args, partition, Partition),
+%%    apply_opts(Rest, NewArgs);
+%%apply_opts([{_, _} | Rest], Args) ->
+%%    % Ignore unknown options
+%%    apply_opts(Rest, Args).
 
 
 doc_member(Cursor, DocProps) ->
@@ -448,13 +467,12 @@ doc_member(Cursor, DocProps) ->
 
 
 match_doc(Selector, Doc, ExecutionStats) ->
-    {ok, Doc, {execution_stats, ExecutionStats}}.
-%%    case mango_selector:match(Selector, Doc) of
-%%        true ->
-%%            {ok, Doc, {execution_stats, ExecutionStats}};
-%%        false ->
-%%            {no_match, Doc, {execution_stats, ExecutionStats}}
-%%    end.
+    case mango_selector:match(Selector, Doc) of
+        true ->
+            {ok, Doc, {execution_stats, ExecutionStats}};
+        false ->
+            {no_match, Doc, {execution_stats, ExecutionStats}}
+    end.
 
 
 is_design_doc(RowProps) ->
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index 36cb40084..1cde268b7 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -38,8 +38,9 @@ query(Db, CallBack, Cursor, Args) ->
             db => TxDb,
             callback => CallBack
         },
-        io:format("DB ~p ~n", [TxDb]),
-        Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, []),
+
+        Opts = args_to_fdb_opts(Args),
+        Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, Opts),
         #{
             cursor := Cursor1
         } = Acc1,
@@ -47,6 +48,71 @@ query(Db, CallBack, Cursor, Args) ->
     end).
 
 
+args_to_fdb_opts(Args) ->
+    #{
+        start_key := StartKey0,
+        start_key_docid := StartKeyDocId,
+        end_key := EndKey0,
+        end_key_docid := EndKeyDocId,
+        direction := Direction,
+        skip := Skip
+    } = Args,
+
+    StartKey1 = if StartKey0 == undefined -> undefined; true ->
+        couch_views_encoding:encode(StartKey0, key)
+    end,
+
+    StartKeyOpts = case {StartKey1, StartKeyDocId} of
+        {undefined, _} ->
+            [];
+        {StartKey1, StartKeyDocId} ->
+            [{start_key, {StartKey1, StartKeyDocId}}]
+    end,
+
+    {EndKey1, InclusiveEnd} = get_endkey_inclusive(EndKey0),
+
+    EndKeyOpts = case {EndKey1, EndKeyDocId, Direction} of
+        {undefined, _, _} ->
+            [];
+        {EndKey1, <<>>, rev} when not InclusiveEnd ->
+            % When we iterate in reverse with
+            % inclusive_end=false we have to set the
+            % EndKeyDocId to <<255>> so that we don't
+            % include matching rows.
+            [{end_key_gt, {EndKey1, <<255>>}}];
+        {EndKey1, <<255>>, _} when not InclusiveEnd ->
+            % When inclusive_end=false we need to
+            % elide the default end_key_docid so as
+            % to not sort past the docids with the
+            % given end key.
+            [{end_key_gt, {EndKey1}}];
+        {EndKey1, EndKeyDocId, _} when not InclusiveEnd ->
+            [{end_key_gt, {EndKey1, EndKeyDocId}}];
+        {EndKey1, EndKeyDocId, _} when InclusiveEnd ->
+            [{end_key, {EndKey1, EndKeyDocId}}]
+    end,
+    [
+        {skip, Skip},
+        {dir, Direction},
+        {streaming_mode, want_all}
+    ] ++ StartKeyOpts ++ EndKeyOpts.
+
+
+get_endkey_inclusive(undefined) ->
+    {undefined, true};
+
+get_endkey_inclusive(EndKey) when is_list(EndKey) ->
+    {EndKey1, InclusiveEnd} = case lists:member(less_than, EndKey) of
+        false ->
+            {EndKey, true};
+        true ->
+            Filtered = lists:filter(fun (Key) -> Key /= less_than end, EndKey),
+            io:format("FIL be ~p after ~p ~n", [EndKey, Filtered]),
+            {Filtered, false}
+    end,
+    {couch_views_encoding:encode(EndKey1, key), InclusiveEnd}.
+
+
 fold_cb({Key, _}, Acc) ->
     #{
         prefix := MangoIdxPrefix,
@@ -55,7 +121,7 @@ fold_cb({Key, _}, Acc) ->
         cursor := Cursor
 
     } = Acc,
-    {_, DocId} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
+    {{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
     {ok, Doc} = fabric2_db:open_doc(Db, DocId),
     io:format("PRINT ~p ~p ~n", [DocId, Doc]),
     {ok, Cursor1} = Callback(Doc, Cursor),
@@ -88,6 +154,6 @@ add_key(TxDb, MangoIdxPrefix, Results, DocId) ->
         tx := Tx
     } = TxDb,
     EncodedResults = couch_views_encoding:encode(Results, key),
-    Key = erlfdb_tuple:pack({EncodedResults, DocId}, MangoIdxPrefix),
+    Key = erlfdb_tuple:pack({{EncodedResults, DocId}}, MangoIdxPrefix),
     erlfdb:set(Tx, Key, <<0>>).
 
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 37911498c..f960ef70d 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -176,7 +176,8 @@ end_key([]) ->
 end_key([{_, _, '$lt', Key} | Rest]) ->
     case mango_json:special(Key) of
         true ->
-            [?MAX_JSON_OBJ];
+%%            [?MAX_JSON_OBJ];
+              [less_than];
         false ->
             [Key | end_key(Rest)]
     end;
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
index 0d213e56e..f1acc673c 100644
--- a/src/mango/src/mango_idx_view.hrl
+++ b/src/mango/src/mango_idx_view.hrl
@@ -10,4 +10,5 @@
 % License for the specific language governing permissions and limitations under
 % the License.
 
--define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
\ No newline at end of file
+%%-define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
+-define(MAX_JSON_OBJ, less_than).
diff --git a/src/mango/src/mango_indexer.erl b/src/mango/src/mango_indexer.erl
index 204005904..36eb2d3b7 100644
--- a/src/mango/src/mango_indexer.erl
+++ b/src/mango/src/mango_indexer.erl
@@ -37,7 +37,7 @@ update(Db, updated, Doc, OldDoc) ->
 update(Db, created, Doc, _) ->
     #doc{id = DocId} = Doc,
     Indexes = mango_idx:list(Db),
-    Indexes1 = filter_and_to_json(Indexes),
+    Indexes1 = filter_json_indexes(Indexes),
     io:format("UPDATE INDEXES ~p ~n filtered ~p ~n", [Indexes, Indexes1]),
     JSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
     io:format("DOC ~p ~n", [Doc]),
@@ -46,12 +46,9 @@ update(Db, created, Doc, _) ->
     mango_fdb:write_doc(Db, DocId, Results).
 
 
-filter_and_to_json(Indexes) ->
+filter_json_indexes(Indexes) ->
     lists:filter(fun (Idx) ->
-        case Idx#idx.type == <<"special">> of
-            true -> false;
-            false -> true
-        end
+        Idx#idx.type == <<"json">>
     end, Indexes).
 
 
diff --git a/src/mango/src/mango_json_bookmark.erl b/src/mango/src/mango_json_bookmark.erl
index 97f81cfb8..edc83cfc0 100644
--- a/src/mango/src/mango_json_bookmark.erl
+++ b/src/mango/src/mango_json_bookmark.erl
@@ -23,23 +23,28 @@
 -include("mango_cursor.hrl").
 -include("mango.hrl").
 
-update_args(EncodedBookmark,  #mrargs{skip = Skip} = Args) ->
+update_args(EncodedBookmark, FdbOpts) ->
+    #{
+        skip := Skip
+    } = FdbOpts,
     Bookmark = unpack(EncodedBookmark),
     case is_list(Bookmark) of
         true -> 
             {startkey, Startkey} = lists:keyfind(startkey, 1, Bookmark),
-            {startkey_docid, StartkeyDocId} = lists:keyfind(startkey_docid, 1, Bookmark),
-            Args#mrargs{
-                start_key = Startkey,
-                start_key_docid = StartkeyDocId,
-                skip = 1 + Skip
+            {startkey_docid, StartkeyDocId} = lists:keyfind(startkey_docid, 1,
+                Bookmark),
+            FdbOpts#{
+                start_key => Startkey,
+                start_key_docid => StartkeyDocId,
+                skip => 1 + Skip
             };
         false ->
-            Args
+            FdbOpts
     end.
     
 
-create(#cursor{bookmark_docid = BookmarkDocId, bookmark_key = BookmarkKey}) when BookmarkKey =/= undefined ->
+create(#cursor{bookmark_docid = BookmarkDocId, bookmark_key = BookmarkKey})
+        when BookmarkKey =/= undefined ->
     QueryArgs = [
         {startkey_docid, BookmarkDocId},
         {startkey, BookmarkKey}
@@ -61,7 +66,8 @@ unpack(Packed) ->
     end.
 
 verify(Bookmark) when is_list(Bookmark) ->
-    case lists:keymember(startkey, 1, Bookmark) andalso lists:keymember(startkey_docid, 1, Bookmark) of
+    case lists:keymember(startkey, 1, Bookmark)
+            andalso lists:keymember(startkey_docid, 1, Bookmark) of
         true -> Bookmark;
         _ -> throw(invalid_bookmark)
     end;
diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py
index 632ad4f4f..8c2a9b1f7 100644
--- a/src/mango/test/02-basic-find-test.py
+++ b/src/mango/test/02-basic-find-test.py
@@ -118,16 +118,13 @@ class BasicFindTests(mango.UserDocsTests):
     #             assert e.response.status_code == 400
     #         else:
     #             raise AssertionError("bad find")
-    #
+
     def test_simple_find(self):
-        print("OK")
-        docs = self.db.find({"age": {"$lt": 45}})
-        print("DOC")
-        print(docs)
+        docs = self.db.find({"age": {"$lt": 35}})
         assert len(docs) == 3
-        # assert docs[0]["user_id"] == 9
-        # assert docs[1]["user_id"] == 1
-        # assert docs[2]["user_id"] == 7
+        assert docs[0]["user_id"] == 9
+        assert docs[1]["user_id"] == 1
+        assert docs[2]["user_id"] == 7
 
     # def test_multi_cond_and(self):
     #     docs = self.db.find({"manager": True, "location.city": "Longbranch"})
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 6fbbb07ed..5ce4219b2 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -110,7 +110,6 @@ class Database(object):
     def save_docs(self, docs, **kwargs):
         body = json.dumps({"docs": docs})
         r = self.sess.post(self.path("_bulk_docs"), data=body, params=kwargs)
-        print(r.json())
         r.raise_for_status()
         for doc, result in zip(docs, r.json()):
             doc["_id"] = result["id"]
diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py
index 45fbd24d1..c021f66bc 100644
--- a/src/mango/test/user_docs.py
+++ b/src/mango/test/user_docs.py
@@ -65,9 +65,9 @@ def setup(db, index_type="view", **kwargs):
         add_view_indexes(db, kwargs)
     elif index_type == "text":
         add_text_indexes(db, kwargs)
-    copy_docs = copy.deepcopy(DOCS)
-    resp = db.save_doc(copy_docs[0])
-    # db.save_docs(copy.deepcopy(DOCS))
+    # copy_docs = copy.deepcopy(DOCS)
+    # resp = db.save_doc(copy_docs[0])
+    db.save_docs(copy.deepcopy(DOCS))
 
 
 def add_view_indexes(db, kwargs):