diff options
author | Garren Smith <garren.smith@gmail.com> | 2020-01-29 11:06:38 +0200 |
---|---|---|
committer | Garren Smith <garren.smith@gmail.com> | 2020-02-17 14:04:49 +0200 |
commit | f131c375f6149e675068925c60bb6a189e09c5f1 (patch) | |
tree | 08c521473268148f28ea301480b33f4cc5a11759 | |
parent | ff80c65a3f14202e639f65a70059cceac8423658 (diff) | |
download | couchdb-f131c375f6149e675068925c60bb6a189e09c5f1.tar.gz |
index and _all_docs queries working
-rw-r--r-- | src/fabric/src/fabric2_db.erl | 23 | ||||
-rw-r--r-- | src/mango/src/mango_cursor_view.erl | 71 | ||||
-rw-r--r-- | src/mango/src/mango_fdb.erl | 128 | ||||
-rw-r--r-- | src/mango/src/mango_idx.erl | 19 | ||||
-rw-r--r-- | src/mango/src/mango_idx_view.erl | 5 | ||||
-rw-r--r-- | src/mango/src/mango_idx_view.hrl | 2 | ||||
-rw-r--r-- | src/mango/test/02-basic-find-test.py | 548 | ||||
-rw-r--r-- | src/mango/test/user_docs.py | 36 |
8 files changed, 439 insertions, 393 deletions
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl index b0f7849e2..7e08e56bd 100644 --- a/src/fabric/src/fabric2_db.erl +++ b/src/fabric/src/fabric2_db.erl @@ -799,11 +799,30 @@ fold_docs(Db, UserFun, UserAcc0, Options) -> UserAcc2 = fabric2_fdb:fold_range(TxDb, Prefix, fun({K, V}, Acc) -> {DocId} = erlfdb_tuple:unpack(K, Prefix), RevId = erlfdb_tuple:unpack(V), - maybe_stop(UserFun({row, [ + Row0 = [ {id, DocId}, {key, DocId}, {value, {[{rev, couch_doc:rev_to_str(RevId)}]}} - ]}, Acc)) + ], + + DocOpts = couch_util:get_value(doc_opts, Options, []), + OpenOpts = [deleted | DocOpts], + + Row1 = case lists:keyfind(include_docs, 1, Options) of + {include_docs, true} -> + DocMember = case fabric2_db:open_doc(Db, DocId, OpenOpts) of + {not_found, missing} -> + []; + {ok, #doc{deleted = true}} -> + [{doc, null}]; + {ok, #doc{} = Doc} -> + [{doc, couch_doc:to_json_obj(Doc, DocOpts)}] + end, + Row0 ++ DocMember; + _ -> Row0 + end, + + maybe_stop(UserFun({row, Row1}, Acc)) end, UserAcc1, Options), {ok, maybe_stop(UserFun(complete, UserAcc2))} diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index 57cab21f5..26c60b7e3 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -63,23 +63,25 @@ explain(Cursor) -> #cursor{ opts = Opts } = Cursor, - [{marargs, {[]}}]. - -%% BaseArgs = base_args(Cursor), -%% Args = apply_opts(Opts, BaseArgs), -%% -%% [{mrargs, {[ -%% {include_docs, Args#mrargs.include_docs}, -%% {view_type, Args#mrargs.view_type}, -%% {reduce, Args#mrargs.reduce}, -%% {partition, couch_mrview_util:get_extra(Args, partition, null)}, -%% {start_key, maybe_replace_max_json(Args#mrargs.start_key)}, -%% {end_key, maybe_replace_max_json(Args#mrargs.end_key)}, -%% {direction, Args#mrargs.direction}, -%% {stable, Args#mrargs.stable}, -%% {update, Args#mrargs.update}, -%% {conflicts, Args#mrargs.conflicts} -%% ]}}]. + + #{ + start_key := StartKey, + end_key := EndKey, + dir := Direction + } = index_args(Cursor), + + [{args, {[ + {include_docs, true}, + {view_type, <<"fdb">>}, + {reduce, false}, + {partition, false}, + {start_key, maybe_replace_max_json(StartKey)}, + {end_key, maybe_replace_max_json(EndKey)}, + {direction, Direction}, + {stable, false}, + {update, true}, + {conflicts, false} + ]}}]. % replace internal values that cannot @@ -100,25 +102,27 @@ maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) -> maybe_replace_max_json(EndKey) -> EndKey. -index_args(#cursor{index = Idx} = Cursor) -> +index_args(#cursor{} = Cursor) -> #cursor{ index = Idx, opts = Opts, bookmark = Bookmark } = Cursor, + io:format("SELE ~p ranges ~p ~n", [Cursor#cursor.selector, Cursor#cursor.ranges]), Args0 = #{ start_key => mango_idx:start_key(Idx, Cursor#cursor.ranges), start_key_docid => <<>>, end_key => mango_idx:end_key(Idx, Cursor#cursor.ranges), - end_key_docid => <<>>, + end_key_docid => <<255>>, skip => 0 }, Args = mango_json_bookmark:update_args(Bookmark, Args0), Sort = couch_util:get_value(sort, Opts, [<<"asc">>]), + io:format("SORT ~p ~n", [Sort]), Args1 = case mango_sort:directions(Sort) of - [<<"desc">> | _] -> Args#{direction => rev}; - _ -> Args#{direction => fwd} + [<<"desc">> | _] -> Args#{dir => rev}; + _ -> Args#{dir => fwd} end, %% TODO: When supported, handle: @@ -140,21 +144,16 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu _ -> Args = index_args(Cursor), #cursor{opts = Opts, bookmark = Bookmark} = Cursor, -%% Args0 = BaseArgs, -%% Args0 = apply_opts(Opts, BaseArgs), -%% Args = mango_json_bookmark:update_args(Bookmark, Args0), UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}), DbOpts = [{user_ctx, UserCtx}], Result = case mango_idx:def(Idx) of all_docs -> CB = fun ?MODULE:handle_all_docs_message/2, - ok; -%% fabric:all_docs(Db, DbOpts, CB, Cursor, Args); + % all_docs + mango_fdb:query_all_docs(Db, CB, Cursor, Args); _ -> CB = fun ?MODULE:handle_message/2, % Normal view -%% DDoc = ddocid(Idx), -%% Name = mango_idx:name(Idx), mango_fdb:query(Db, CB, Cursor, Args) end, case Result of @@ -303,16 +302,15 @@ set_mango_msg_timestamp() -> handle_message({meta, _}, Cursor) -> {ok, Cursor}; -handle_message(Doc, Cursor) -> - JSONDoc = couch_doc:to_json_obj(Doc, []), - case doc_member(Cursor, JSONDoc) of - {ok, JSONDoc, {execution_stats, ExecutionStats1}} -> +handle_message({doc, Doc}, Cursor) -> + case doc_member(Cursor, Doc) of + {ok, Doc, {execution_stats, ExecutionStats1}} -> Cursor1 = Cursor#cursor { execution_stats = ExecutionStats1 }, - {Props} = JSONDoc, + {Props} = Doc, Cursor2 = update_bookmark_keys(Cursor1, Props), - FinalDoc = mango_fields:extract(JSONDoc, Cursor2#cursor.fields), + FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields), handle_doc(Cursor2, FinalDoc); {no_match, _, {execution_stats, ExecutionStats1}} -> Cursor1 = Cursor#cursor { @@ -330,9 +328,12 @@ handle_message({error, Reason}, _Cursor) -> handle_all_docs_message({row, Props}, Cursor) -> + io:format("ALL DOCS ~p ~n", [Props]), case is_design_doc(Props) of true -> {ok, Cursor}; - false -> handle_message({row, Props}, Cursor) + false -> + {doc, Doc} = lists:keyfind(doc, 1, Props), + handle_message({doc, Doc}, Cursor) end; handle_all_docs_message(Message, Cursor) -> handle_message(Message, Cursor). diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl index 1cde268b7..091d5f7c1 100644 --- a/src/mango/src/mango_fdb.erl +++ b/src/mango/src/mango_fdb.erl @@ -18,14 +18,21 @@ -include("mango.hrl"). -include("mango_idx.hrl"). -include("mango_cursor.hrl"). +-include("mango_idx_view.hrl"). -export([ + query_all_docs/4, write_doc/3, query/4 ]). +query_all_docs(Db, CallBack, Cursor, Args) -> + Opts = args_to_fdb_opts(Args) ++ [{include_docs, true}], + fabric2_db:fold_docs(Db, CallBack, Cursor, Opts). + + query(Db, CallBack, Cursor, Args) -> #cursor{ index = Idx @@ -40,11 +47,17 @@ query(Db, CallBack, Cursor, Args) -> }, Opts = args_to_fdb_opts(Args), - Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, Opts), - #{ - cursor := Cursor1 - } = Acc1, - {ok, Cursor1} + io:format("OPTS ~p ~n", [Opts]), + try + Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, Opts), + #{ + cursor := Cursor1 + } = Acc1, + {ok, Cursor1} + catch + throw:{stop, StopCursor} -> + {ok, StopCursor} + end end). @@ -54,43 +67,66 @@ args_to_fdb_opts(Args) -> start_key_docid := StartKeyDocId, end_key := EndKey0, end_key_docid := EndKeyDocId, - direction := Direction, + dir := Direction, skip := Skip } = Args, - StartKey1 = if StartKey0 == undefined -> undefined; true -> - couch_views_encoding:encode(StartKey0, key) + io:format("ARGS ~p ~n", [Args]), + io:format("START ~p ~n End ~p ~n", [StartKey0, EndKey0]), +%% StartKey1 = if StartKey0 == undefined -> undefined; true -> +%% couch_views_encoding:encode(StartKey0, key) +%% end, + + % fabric2_fdb:fold_range switches keys around because map/reduce switches them + % but we do need to switch them. So we do this fun dance + {StartKeyName, EndKeyName} = case Direction of + rev -> {end_key, start_key}; + _ -> {start_key, end_key} end, - StartKeyOpts = case {StartKey1, StartKeyDocId} of - {undefined, _} -> + StartKeyOpts = case {StartKey0, StartKeyDocId} of + {[], _} -> []; - {StartKey1, StartKeyDocId} -> - [{start_key, {StartKey1, StartKeyDocId}}] + {null, _} -> + %% all_docs no startkey + []; +%% {undefined, _} -> +%% []; + {StartKey0, StartKeyDocId} -> + StartKey1 = couch_views_encoding:encode(StartKey0, key), + [{StartKeyName, {StartKey1, StartKeyDocId}}] end, - {EndKey1, InclusiveEnd} = get_endkey_inclusive(EndKey0), + InclusiveEnd = true, - EndKeyOpts = case {EndKey1, EndKeyDocId, Direction} of - {undefined, _, _} -> + EndKeyOpts = case {EndKey0, EndKeyDocId, Direction} of + {<<255>>, _, _} -> + %% all_docs no endkey + []; + {[<<255>>], _, _} -> + %% mango index no endkey []; - {EndKey1, <<>>, rev} when not InclusiveEnd -> - % When we iterate in reverse with - % inclusive_end=false we have to set the - % EndKeyDocId to <<255>> so that we don't - % include matching rows. - [{end_key_gt, {EndKey1, <<255>>}}]; - {EndKey1, <<255>>, _} when not InclusiveEnd -> - % When inclusive_end=false we need to - % elide the default end_key_docid so as - % to not sort past the docids with the - % given end key. - [{end_key_gt, {EndKey1}}]; - {EndKey1, EndKeyDocId, _} when not InclusiveEnd -> - [{end_key_gt, {EndKey1, EndKeyDocId}}]; - {EndKey1, EndKeyDocId, _} when InclusiveEnd -> - [{end_key, {EndKey1, EndKeyDocId}}] +%% {undefined, _, _} -> +%% []; +%% {EndKey1, <<>>, rev} when not InclusiveEnd -> +%% % When we iterate in reverse with +%% % inclusive_end=false we have to set the +%% % EndKeyDocId to <<255>> so that we don't +%% % include matching rows. +%% [{end_key_gt, {EndKey1, <<255>>}}]; +%% {EndKey1, <<255>>, _} when not InclusiveEnd -> +%% % When inclusive_end=false we need to +%% % elide the default end_key_docid so as +%% % to not sort past the docids with the +%% % given end key. +%% [{end_key_gt, {EndKey1}}]; +%% {EndKey1, EndKeyDocId, _} when not InclusiveEnd -> +%% [{end_key_gt, {EndKey1, EndKeyDocId}}]; + {EndKey0, EndKeyDocId, _} when InclusiveEnd -> + EndKey1 = couch_views_encoding:encode(EndKey0, key), + [{EndKeyName, {EndKey1, EndKeyDocId}}] end, + [ {skip, Skip}, {dir, Direction}, @@ -98,21 +134,6 @@ args_to_fdb_opts(Args) -> ] ++ StartKeyOpts ++ EndKeyOpts. -get_endkey_inclusive(undefined) -> - {undefined, true}; - -get_endkey_inclusive(EndKey) when is_list(EndKey) -> - {EndKey1, InclusiveEnd} = case lists:member(less_than, EndKey) of - false -> - {EndKey, true}; - true -> - Filtered = lists:filter(fun (Key) -> Key /= less_than end, EndKey), - io:format("FIL be ~p after ~p ~n", [EndKey, Filtered]), - {Filtered, false} - end, - {couch_views_encoding:encode(EndKey1, key), InclusiveEnd}. - - fold_cb({Key, _}, Acc) -> #{ prefix := MangoIdxPrefix, @@ -123,11 +144,16 @@ fold_cb({Key, _}, Acc) -> } = Acc, {{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix), {ok, Doc} = fabric2_db:open_doc(Db, DocId), - io:format("PRINT ~p ~p ~n", [DocId, Doc]), - {ok, Cursor1} = Callback(Doc, Cursor), - Acc#{ - cursor := Cursor1 - }. + JSONDoc = couch_doc:to_json_obj(Doc, []), + io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]), + case Callback({doc, JSONDoc}, Cursor) of + {ok, Cursor1} -> + Acc#{ + cursor := Cursor1 + }; + {stop, Cursor1} -> + throw({stop, Cursor1}) + end. write_doc(TxDb, DocId, IdxResults) -> diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl index b9ce6402a..57262f963 100644 --- a/src/mango/src/mango_idx.erl +++ b/src/mango/src/mango_idx.erl @@ -109,14 +109,16 @@ get_usable_indexes(Db, Selector, Opts) -> mango_sort_error(Db, Opts) -> - case {fabric_util:is_partitioned(Db), is_opts_partitioned(Opts)} of - {false, _} -> - ?MANGO_ERROR({no_usable_index, missing_sort_index}); - {true, true} -> - ?MANGO_ERROR({no_usable_index, missing_sort_index_partitioned}); - {true, false} -> - ?MANGO_ERROR({no_usable_index, missing_sort_index_global}) - end. + ?MANGO_ERROR({no_usable_index, missing_sort_index}). +% TODO: add back in when partitions supported +%% case {fabric_util:is_partitioned(Db), is_opts_partitioned(Opts)} of +%% {false, _} -> +%% ?MANGO_ERROR({no_usable_index, missing_sort_index}); +%% {true, true} -> +%% ?MANGO_ERROR({no_usable_index, missing_sort_index_partitioned}); +%% {true, false} -> +%% ?MANGO_ERROR({no_usable_index, missing_sort_index_global}) +%% end. recover(Db) -> @@ -291,6 +293,7 @@ start_key(#idx{}=Idx, Ranges) -> end_key(#idx{}=Idx, Ranges) -> Mod = idx_mod(Idx), + io:format("END KEY ~p ~n", [Mod]), Mod:end_key(Ranges). diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl index f960ef70d..5ec2a1020 100644 --- a/src/mango/src/mango_idx_view.erl +++ b/src/mango/src/mango_idx_view.erl @@ -172,12 +172,11 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) -> end_key([]) -> - [?MAX_JSON_OBJ]; + []; end_key([{_, _, '$lt', Key} | Rest]) -> case mango_json:special(Key) of true -> -%% [?MAX_JSON_OBJ]; - [less_than]; + [?MAX_JSON_OBJ]; false -> [Key | end_key(Rest)] end; diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl index f1acc673c..a6fc2b40d 100644 --- a/src/mango/src/mango_idx_view.hrl +++ b/src/mango/src/mango_idx_view.hrl @@ -11,4 +11,4 @@ % the License. %%-define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}). --define(MAX_JSON_OBJ, less_than). +-define(MAX_JSON_OBJ, <<255>>). diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py index 8c2a9b1f7..fd66e302b 100644 --- a/src/mango/test/02-basic-find-test.py +++ b/src/mango/test/02-basic-find-test.py @@ -16,108 +16,108 @@ import mango class BasicFindTests(mango.UserDocsTests): - # def test_bad_selector(self): - # bad_selectors = [ - # None, - # True, - # False, - # 1.0, - # "foobarbaz", - # {"foo": {"$not_an_op": 2}}, - # {"$gt": 2}, - # [None, "bing"], - # ] - # for bs in bad_selectors: - # try: - # self.db.find(bs) - # except Exception as e: - # assert e.response.status_code == 400 - # else: - # raise AssertionError("bad find") - # - # def test_bad_limit(self): - # bad_limits = ([None, True, False, -1, 1.2, "no limit!", {"foo": "bar"}, [2]],) - # for bl in bad_limits: - # try: - # self.db.find({"int": {"$gt": 2}}, limit=bl) - # except Exception as e: - # assert e.response.status_code == 400 - # else: - # raise AssertionError("bad find") - # - # def test_bad_skip(self): - # bad_skips = ([None, True, False, -3, 1.2, "no limit!", {"foo": "bar"}, [2]],) - # for bs in bad_skips: - # try: - # self.db.find({"int": {"$gt": 2}}, skip=bs) - # except Exception as e: - # assert e.response.status_code == 400 - # else: - # raise AssertionError("bad find") - # - # def test_bad_sort(self): - # bad_sorts = ( - # [ - # None, - # True, - # False, - # 1.2, - # "no limit!", - # {"foo": "bar"}, - # [2], - # [{"foo": "asc", "bar": "asc"}], - # [{"foo": "asc"}, {"bar": "desc"}], - # ], - # ) - # for bs in bad_sorts: - # try: - # self.db.find({"int": {"$gt": 2}}, sort=bs) - # except Exception as e: - # assert e.response.status_code == 400 - # else: - # raise AssertionError("bad find") - # - # def test_bad_fields(self): - # bad_fields = ( - # [ - # None, - # True, - # False, - # 1.2, - # "no limit!", - # {"foo": "bar"}, - # [2], - # [[]], - # ["foo", 2.0], - # ], - # ) - # for bf in bad_fields: - # try: - # self.db.find({"int": {"$gt": 2}}, fields=bf) - # except Exception as e: - # assert e.response.status_code == 400 - # else: - # raise AssertionError("bad find") - # - # def test_bad_r(self): - # bad_rs = ([None, True, False, 1.2, "no limit!", {"foo": "bar"}, [2]],) - # for br in bad_rs: - # try: - # self.db.find({"int": {"$gt": 2}}, r=br) - # except Exception as e: - # assert e.response.status_code == 400 - # else: - # raise AssertionError("bad find") - # - # def test_bad_conflicts(self): - # bad_conflicts = ([None, 1.2, "no limit!", {"foo": "bar"}, [2]],) - # for bc in bad_conflicts: - # try: - # self.db.find({"int": {"$gt": 2}}, conflicts=bc) - # except Exception as e: - # assert e.response.status_code == 400 - # else: - # raise AssertionError("bad find") + def test_bad_selector(self): + bad_selectors = [ + None, + True, + False, + 1.0, + "foobarbaz", + {"foo": {"$not_an_op": 2}}, + {"$gt": 2}, + [None, "bing"], + ] + for bs in bad_selectors: + try: + self.db.find(bs) + except Exception as e: + assert e.response.status_code == 400 + else: + raise AssertionError("bad find") + + def test_bad_limit(self): + bad_limits = ([None, True, False, -1, 1.2, "no limit!", {"foo": "bar"}, [2]],) + for bl in bad_limits: + try: + self.db.find({"int": {"$gt": 2}}, limit=bl) + except Exception as e: + assert e.response.status_code == 400 + else: + raise AssertionError("bad find") + + def test_bad_skip(self): + bad_skips = ([None, True, False, -3, 1.2, "no limit!", {"foo": "bar"}, [2]],) + for bs in bad_skips: + try: + self.db.find({"int": {"$gt": 2}}, skip=bs) + except Exception as e: + assert e.response.status_code == 400 + else: + raise AssertionError("bad find") + + def test_bad_sort(self): + bad_sorts = ( + [ + None, + True, + False, + 1.2, + "no limit!", + {"foo": "bar"}, + [2], + [{"foo": "asc", "bar": "asc"}], + [{"foo": "asc"}, {"bar": "desc"}], + ], + ) + for bs in bad_sorts: + try: + self.db.find({"int": {"$gt": 2}}, sort=bs) + except Exception as e: + assert e.response.status_code == 400 + else: + raise AssertionError("bad find") + + def test_bad_fields(self): + bad_fields = ( + [ + None, + True, + False, + 1.2, + "no limit!", + {"foo": "bar"}, + [2], + [[]], + ["foo", 2.0], + ], + ) + for bf in bad_fields: + try: + self.db.find({"int": {"$gt": 2}}, fields=bf) + except Exception as e: + assert e.response.status_code == 400 + else: + raise AssertionError("bad find") + + def test_bad_r(self): + bad_rs = ([None, True, False, 1.2, "no limit!", {"foo": "bar"}, [2]],) + for br in bad_rs: + try: + self.db.find({"int": {"$gt": 2}}, r=br) + except Exception as e: + assert e.response.status_code == 400 + else: + raise AssertionError("bad find") + + def test_bad_conflicts(self): + bad_conflicts = ([None, 1.2, "no limit!", {"foo": "bar"}, [2]],) + for bc in bad_conflicts: + try: + self.db.find({"int": {"$gt": 2}}, conflicts=bc) + except Exception as e: + assert e.response.status_code == 400 + else: + raise AssertionError("bad find") def test_simple_find(self): docs = self.db.find({"age": {"$lt": 35}}) @@ -126,176 +126,176 @@ class BasicFindTests(mango.UserDocsTests): assert docs[1]["user_id"] == 1 assert docs[2]["user_id"] == 7 - # def test_multi_cond_and(self): - # docs = self.db.find({"manager": True, "location.city": "Longbranch"}) - # assert len(docs) == 1 - # assert docs[0]["user_id"] == 7 - # - # def test_multi_cond_duplicate_field(self): - # # need to explicitly define JSON as dict won't allow duplicate keys - # body = ( - # '{"selector":{"location.city":{"$regex": "^L+"},' - # '"location.city":{"$exists":true}}}' - # ) - # r = self.db.sess.post(self.db.path("_find"), data=body) - # r.raise_for_status() - # docs = r.json()["docs"] - # - # # expectation is that only the second instance - # # of the "location.city" field is used - # self.assertEqual(len(docs), 15) - # - # def test_multi_cond_or(self): - # docs = self.db.find( - # { - # "$and": [ - # {"age": {"$gte": 75}}, - # {"$or": [{"name.first": "Mathis"}, {"name.first": "Whitley"}]}, - # ] - # } - # ) - # assert len(docs) == 2 - # assert docs[0]["user_id"] == 11 - # assert docs[1]["user_id"] == 13 - # - # def test_multi_col_idx(self): - # docs = self.db.find( - # { - # "location.state": {"$and": [{"$gt": "Hawaii"}, {"$lt": "Maine"}]}, - # "location.city": {"$lt": "Longbranch"}, - # } - # ) - # assert len(docs) == 1 - # assert docs[0]["user_id"] == 6 - # - # def test_missing_not_indexed(self): - # docs = self.db.find({"favorites.3": "C"}) - # assert len(docs) == 1 - # assert docs[0]["user_id"] == 6 - # - # docs = self.db.find({"favorites.3": None}) - # assert len(docs) == 0 - # - # docs = self.db.find({"twitter": {"$gt": None}}) - # assert len(docs) == 4 - # assert docs[0]["user_id"] == 1 - # assert docs[1]["user_id"] == 4 - # assert docs[2]["user_id"] == 0 - # assert docs[3]["user_id"] == 13 - # - # def test_limit(self): - # docs = self.db.find({"age": {"$gt": 0}}) - # assert len(docs) == 15 - # for l in [0, 1, 5, 14]: - # docs = self.db.find({"age": {"$gt": 0}}, limit=l) - # assert len(docs) == l - # - # def test_skip(self): - # docs = self.db.find({"age": {"$gt": 0}}) - # assert len(docs) == 15 - # for s in [0, 1, 5, 14]: - # docs = self.db.find({"age": {"$gt": 0}}, skip=s) - # assert len(docs) == (15 - s) - # - # def test_sort(self): - # docs1 = self.db.find({"age": {"$gt": 0}}, sort=[{"age": "asc"}]) - # docs2 = list(sorted(docs1, key=lambda d: d["age"])) - # assert docs1 is not docs2 and docs1 == docs2 - # - # docs1 = self.db.find({"age": {"$gt": 0}}, sort=[{"age": "desc"}]) - # docs2 = list(reversed(sorted(docs1, key=lambda d: d["age"]))) - # assert docs1 is not docs2 and docs1 == docs2 - # - # def test_sort_desc_complex(self): - # docs = self.db.find( - # { - # "company": {"$lt": "M"}, - # "$or": [{"company": "Dreamia"}, {"manager": True}], - # }, - # sort=[{"company": "desc"}, {"manager": "desc"}], - # ) - # - # companies_returned = list(d["company"] for d in docs) - # desc_companies = sorted(companies_returned, reverse=True) - # self.assertEqual(desc_companies, companies_returned) - # - # def test_sort_with_primary_sort_not_in_selector(self): - # try: - # docs = self.db.find( - # {"name.last": {"$lt": "M"}}, sort=[{"name.first": "desc"}] - # ) - # except Exception as e: - # self.assertEqual(e.response.status_code, 400) - # resp = e.response.json() - # self.assertEqual(resp["error"], "no_usable_index") - # else: - # raise AssertionError("expected find error") - # - # def test_sort_exists_true(self): - # docs1 = self.db.find( - # {"age": {"$gt": 0, "$exists": True}}, sort=[{"age": "asc"}] - # ) - # docs2 = list(sorted(docs1, key=lambda d: d["age"])) - # assert docs1 is not docs2 and docs1 == docs2 - # - # def test_sort_desc_complex_error(self): - # try: - # self.db.find( - # { - # "company": {"$lt": "M"}, - # "$or": [{"company": "Dreamia"}, {"manager": True}], - # }, - # sort=[{"company": "desc"}], - # ) - # except Exception as e: - # self.assertEqual(e.response.status_code, 400) - # resp = e.response.json() - # self.assertEqual(resp["error"], "no_usable_index") - # else: - # raise AssertionError("expected find error") - # - # def test_fields(self): - # selector = {"age": {"$gt": 0}} - # docs = self.db.find(selector, fields=["user_id", "location.address"]) - # for d in docs: - # assert sorted(d.keys()) == ["location", "user_id"] - # assert sorted(d["location"].keys()) == ["address"] - # - # def test_r(self): - # for r in [1, 2, 3]: - # docs = self.db.find({"age": {"$gt": 0}}, r=r) - # assert len(docs) == 15 - # - # def test_empty(self): - # docs = self.db.find({}) - # # 15 users - # assert len(docs) == 15 - # - # def test_empty_subsel(self): - # docs = self.db.find({"_id": {"$gt": None}, "location": {}}) - # assert len(docs) == 0 - # - # def test_empty_subsel_match(self): - # self.db.save_docs([{"user_id": "eo", "empty_obj": {}}]) - # docs = self.db.find({"_id": {"$gt": None}, "empty_obj": {}}) - # assert len(docs) == 1 - # assert docs[0]["user_id"] == "eo" - # - # def test_unsatisfiable_range(self): - # docs = self.db.find({"$and": [{"age": {"$gt": 0}}, {"age": {"$lt": 0}}]}) - # assert len(docs) == 0 - # - # def test_explain_view_args(self): - # explain = self.db.find({"age": {"$gt": 0}}, fields=["manager"], explain=True) - # assert explain["mrargs"]["stable"] == False - # assert explain["mrargs"]["update"] == True - # assert explain["mrargs"]["reduce"] == False - # assert explain["mrargs"]["start_key"] == [0] - # assert explain["mrargs"]["end_key"] == ["<MAX>"] - # assert explain["mrargs"]["include_docs"] == True + def test_multi_cond_and(self): + docs = self.db.find({"manager": True, "location.city": "Longbranch"}) + assert len(docs) == 1 + assert docs[0]["user_id"] == 7 + + def test_multi_cond_duplicate_field(self): + # need to explicitly define JSON as dict won't allow duplicate keys + body = ( + '{"selector":{"location.city":{"$regex": "^L+"},' + '"location.city":{"$exists":true}}}' + ) + r = self.db.sess.post(self.db.path("_find"), data=body) + r.raise_for_status() + docs = r.json()["docs"] + + # expectation is that only the second instance + # of the "location.city" field is used + self.assertEqual(len(docs), 15) + + def test_multi_cond_or(self): + docs = self.db.find( + { + "$and": [ + {"age": {"$gte": 75}}, + {"$or": [{"name.first": "Mathis"}, {"name.first": "Whitley"}]}, + ] + } + ) + assert len(docs) == 2 + assert docs[0]["user_id"] == 11 + assert docs[1]["user_id"] == 13 + + def test_multi_col_idx(self): + docs = self.db.find( + { + "location.state": {"$and": [{"$gt": "Hawaii"}, {"$lt": "Maine"}]}, + "location.city": {"$lt": "Longbranch"}, + } + ) + assert len(docs) == 1 + assert docs[0]["user_id"] == 6 + + def test_missing_not_indexed(self): + docs = self.db.find({"favorites.3": "C"}) + assert len(docs) == 1 + assert docs[0]["user_id"] == 6 + + docs = self.db.find({"favorites.3": None}) + assert len(docs) == 0 + + docs = self.db.find({"twitter": {"$gt": None}}) + assert len(docs) == 4 + assert docs[0]["user_id"] == 1 + assert docs[1]["user_id"] == 4 + assert docs[2]["user_id"] == 0 + assert docs[3]["user_id"] == 13 + + def test_limit(self): + docs = self.db.find({"age": {"$gt": 0}}) + assert len(docs) == 15 + for l in [0, 1, 5, 14]: + docs = self.db.find({"age": {"$gt": 0}}, limit=l) + assert len(docs) == l + + def test_skip(self): + docs = self.db.find({"age": {"$gt": 0}}) + assert len(docs) == 15 + for s in [0, 1, 5, 14]: + docs = self.db.find({"age": {"$gt": 0}}, skip=s) + assert len(docs) == (15 - s) + + def test_sort(self): + docs1 = self.db.find({"age": {"$gt": 0}}, sort=[{"age": "asc"}]) + docs2 = list(sorted(docs1, key=lambda d: d["age"])) + assert docs1 is not docs2 and docs1 == docs2 + + docs1 = self.db.find({"age": {"$gt": 0}}, sort=[{"age": "desc"}]) + docs2 = list(reversed(sorted(docs1, key=lambda d: d["age"]))) + assert docs1 is not docs2 and docs1 == docs2 # - # def test_sort_with_all_docs(self): - # explain = self.db.find( - # {"_id": {"$gt": 0}, "age": {"$gt": 0}}, sort=["_id"], explain=True - # ) - # self.assertEqual(explain["index"]["type"], "special") + def test_sort_desc_complex(self): + docs = self.db.find( + { + "company": {"$lt": "M"}, + "$or": [{"company": "Dreamia"}, {"manager": True}], + }, + sort=[{"company": "desc"}, {"manager": "desc"}], + ) + + companies_returned = list(d["company"] for d in docs) + desc_companies = sorted(companies_returned, reverse=True) + self.assertEqual(desc_companies, companies_returned) + + def test_sort_with_primary_sort_not_in_selector(self): + try: + docs = self.db.find( + {"name.last": {"$lt": "M"}}, sort=[{"name.first": "desc"}] + ) + except Exception as e: + self.assertEqual(e.response.status_code, 400) + resp = e.response.json() + self.assertEqual(resp["error"], "no_usable_index") + else: + raise AssertionError("expected find error") + + def test_sort_exists_true(self): + docs1 = self.db.find( + {"age": {"$gt": 0, "$exists": True}}, sort=[{"age": "asc"}] + ) + docs2 = list(sorted(docs1, key=lambda d: d["age"])) + assert docs1 is not docs2 and docs1 == docs2 + + def test_sort_desc_complex_error(self): + try: + self.db.find( + { + "company": {"$lt": "M"}, + "$or": [{"company": "Dreamia"}, {"manager": True}], + }, + sort=[{"company": "desc"}], + ) + except Exception as e: + self.assertEqual(e.response.status_code, 400) + resp = e.response.json() + self.assertEqual(resp["error"], "no_usable_index") + else: + raise AssertionError("expected find error") + + def test_fields(self): + selector = {"age": {"$gt": 0}} + docs = self.db.find(selector, fields=["user_id", "location.address"]) + for d in docs: + assert sorted(d.keys()) == ["location", "user_id"] + assert sorted(d["location"].keys()) == ["address"] + + def test_r(self): + for r in [1, 2, 3]: + docs = self.db.find({"age": {"$gt": 0}}, r=r) + assert len(docs) == 15 + + def test_empty(self): + docs = self.db.find({}) + # 15 users + assert len(docs) == 15 + + def test_empty_subsel(self): + docs = self.db.find({"_id": {"$gt": None}, "location": {}}) + assert len(docs) == 0 + + def test_empty_subsel_match(self): + self.db.save_docs([{"user_id": "eo", "empty_obj": {}}]) + docs = self.db.find({"_id": {"$gt": None}, "empty_obj": {}}) + assert len(docs) == 1 + assert docs[0]["user_id"] == "eo" + + def test_unsatisfiable_range(self): + docs = self.db.find({"$and": [{"age": {"$gt": 0}}, {"age": {"$lt": 0}}]}) + assert len(docs) == 0 + + def test_explain_view_args(self): + explain = self.db.find({"age": {"$gt": 0}}, fields=["manager"], explain=True) + assert explain["args"]["stable"] == False + assert explain["args"]["update"] == True + assert explain["args"]["reduce"] == False + assert explain["args"]["start_key"] == [0] + assert explain["args"]["end_key"] == ["<MAX>"] + assert explain["args"]["include_docs"] == True + + def test_sort_with_all_docs(self): + explain = self.db.find( + {"_id": {"$gt": 0}, "age": {"$gt": 0}}, sort=["_id"], explain=True + ) + self.assertEqual(explain["index"]["type"], "special") diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py index c021f66bc..0a52deeb5 100644 --- a/src/mango/test/user_docs.py +++ b/src/mango/test/user_docs.py @@ -65,31 +65,29 @@ def setup(db, index_type="view", **kwargs): add_view_indexes(db, kwargs) elif index_type == "text": add_text_indexes(db, kwargs) - # copy_docs = copy.deepcopy(DOCS) - # resp = db.save_doc(copy_docs[0]) db.save_docs(copy.deepcopy(DOCS)) def add_view_indexes(db, kwargs): indexes = [ - # (["user_id"], "user_id"), - # (["name.last", "name.first"], "name"), + (["user_id"], "user_id"), + (["name.last", "name.first"], "name"), (["age"], "age"), - # ( - # [ - # "location.state", - # "location.city", - # "location.address.street", - # "location.address.number", - # ], - # "location", - # ), - # (["company", "manager"], "company_and_manager"), - # (["manager"], "manager"), - # (["favorites"], "favorites"), - # (["favorites.3"], "favorites_3"), - # (["twitter"], "twitter"), - # (["ordered"], "ordered"), + ( + [ + "location.state", + "location.city", + "location.address.street", + "location.address.number", + ], + "location", + ), + (["company", "manager"], "company_and_manager"), + (["manager"], "manager"), + (["favorites"], "favorites"), + (["favorites.3"], "favorites_3"), + (["twitter"], "twitter"), + (["ordered"], "ordered"), ] for (idx, name) in indexes: assert db.create_index(idx, name=name, ddoc=name) is True |