diff options
author | Garren Smith <garren.smith@gmail.com> | 2020-03-23 16:21:22 +0200 |
---|---|---|
committer | garren smith <garren.smith@gmail.com> | 2020-04-06 17:55:49 +0200 |
commit | 377b0c26c845a5c5152ef493f833889b45625f4a (patch) | |
tree | 459d3cc2b00bf4f8e21a37d8d5082c5728dcab48 | |
parent | c483652fee917ae3c6d4065cc6deb28296e40065 (diff) | |
download | couchdb-377b0c26c845a5c5152ef493f833889b45625f4a.tar.gz |
Update Mango query to work with couch_views
-rw-r--r-- | src/couch_views/test/couch_views_map_test.erl | 41 | ||||
-rw-r--r-- | src/mango/src/mango_crud.erl | 24 | ||||
-rw-r--r-- | src/mango/src/mango_cursor.erl | 4 | ||||
-rw-r--r-- | src/mango/src/mango_cursor_view.erl | 70 | ||||
-rw-r--r-- | src/mango/src/mango_httpd.erl | 105 | ||||
-rw-r--r-- | src/mango/src/mango_idx.erl | 13 | ||||
-rw-r--r-- | src/mango/src/mango_idx_view.erl | 5 | ||||
-rw-r--r-- | src/mango/src/mango_idx_view.hrl | 13 | ||||
-rw-r--r-- | src/mango/src/mango_util.erl | 11 |
9 files changed, 139 insertions, 147 deletions
diff --git a/src/couch_views/test/couch_views_map_test.erl b/src/couch_views/test/couch_views_map_test.erl index f8ba18319..7d1e94b2c 100644 --- a/src/couch_views/test/couch_views_map_test.erl +++ b/src/couch_views/test/couch_views_map_test.erl @@ -14,6 +14,7 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). +-include("couch_views.hrl"). -define(TDEF(A), {atom_to_list(A), fun A/0}). @@ -56,7 +57,8 @@ map_views_test_() -> ?TDEF(should_map_duplicate_keys), ?TDEF(should_map_with_doc_emit), ?TDEF(should_map_update_is_false), - ?TDEF(should_map_update_is_lazy) + ?TDEF(should_map_update_is_lazy), + ?TDEF(should_map_wait_for_interactive) % fun should_give_ext_size_seq_indexed_test/1 ] } @@ -419,6 +421,25 @@ should_map_update_is_lazy() -> ?assertEqual(Expect, Result2). +should_map_wait_for_interactive() -> + DbName = ?tempdb(), + {ok, Db} = fabric2_db:create(DbName, [{user_ctx, ?ADMIN_USER}]), + + DDoc = create_interactive_ddoc(), + Docs = make_docs(101), + + fabric2_db:update_docs(Db, Docs), + fabric2_db:update_docs(Db, [DDoc]), + + Result = couch_views:query(Db, DDoc, <<"idx_01">>, fun default_cb/2, [], + #{limit => 3}), + ?assertEqual({ok, [ + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]} + ]}, Result). + + % should_give_ext_size_seq_indexed_test(Db) -> % DDoc = couch_doc:from_json_obj({[ % {<<"_id">>, <<"_design/seqdoc">>}, @@ -510,6 +531,24 @@ create_ddoc() -> ]}} ]}). +create_interactive_ddoc() -> + couch_doc:from_json_obj({[ + {<<"_id">>, <<"_design/ddoc_interactive">>}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, {[ + {<<"idx_01">>, {[ + {<<"map">>, << + "function(doc) {" + "if (doc.val) {" + "emit(doc.val, doc.val);" + "}" + "}">>} + ]}} + ]}}, + {<<"autoupdate">>, false}, + {<<"interactive">>, true} + ]}). + make_docs(Count) -> [doc(I) || I <- lists:seq(1, Count)]. diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl index 42717ffc8..66cef65b3 100644 --- a/src/mango/src/mango_crud.erl +++ b/src/mango/src/mango_crud.erl @@ -33,9 +33,8 @@ insert(Db, #doc{}=Doc, Opts) -> insert(Db, [Doc], Opts); insert(Db, {_}=Doc, Opts) -> insert(Db, [Doc], Opts); -insert(Db, Docs, Opts0) when is_list(Docs) -> - Opts1 = maybe_add_user_ctx(Db, Opts0), - case fabric:update_docs(Db, Docs, Opts1) of +insert(Db, Docs, Opts) when is_list(Docs) -> + case fabric2_db:update_docs(Db, Docs, Opts) of {ok, Results0} -> {ok, lists:zipwith(fun result_to_json/2, Docs, Results0)}; {accepted, Results0} -> @@ -45,9 +44,8 @@ insert(Db, Docs, Opts0) when is_list(Docs) -> end. -find(Db, Selector, Callback, UserAcc, Opts0) -> - Opts1 = maybe_add_user_ctx(Db, Opts0), - {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1), +find(Db, Selector, Callback, UserAcc, Opts) -> + {ok, Cursor} = mango_cursor:create(Db, Selector, Opts), mango_cursor:execute(Cursor, Callback, UserAcc). @@ -97,21 +95,11 @@ delete(Db, Selector, Options) -> end. -explain(Db, Selector, Opts0) -> - Opts1 = maybe_add_user_ctx(Db, Opts0), - {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1), +explain(Db, Selector, Opts) -> + {ok, Cursor} = mango_cursor:create(Db, Selector, Opts), mango_cursor:explain(Cursor). -maybe_add_user_ctx(Db, Opts) -> - case lists:keyfind(user_ctx, 1, Opts) of - {user_ctx, _} -> - Opts; - false -> - [{user_ctx, couch_db:get_user_ctx(Db)} | Opts] - end. - - result_to_json(#doc{id=Id}, Result) -> result_to_json(Id, Result); result_to_json({Props}, Result) -> diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl index db4e98184..63b449cdc 100644 --- a/src/mango/src/mango_cursor.erl +++ b/src/mango/src/mango_cursor.erl @@ -48,7 +48,9 @@ create(Db, Selector0, Opts) -> Selector = mango_selector:normalize(Selector0), - UsableIndexes = mango_idx:get_usable_indexes(Db, Selector, Opts), + UsableIndexes = fabric2_fdb:transactional(Db, fun (TxDb) -> + mango_idx:get_usable_indexes(TxDb, Selector, Opts) + end), case mango_cursor:maybe_filter_indexes_by_ddoc(UsableIndexes, Opts) of [] -> % use_index doesn't match a valid index - fall back to a valid one diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index b88f6eaee..4960fa126 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -31,9 +31,7 @@ -include_lib("fabric/include/fabric.hrl"). -include("mango_cursor.hrl"). --include("mango_idx_view.hrl"). --define(HEARTBEAT_INTERVAL_IN_USEC, 4000000). create(Db, Indexes, Selector, Opts) -> FieldRanges = mango_idx_view:field_ranges(Selector), @@ -91,7 +89,8 @@ maybe_replace_max_json(?MAX_STR) -> <<"<MAX>">>; maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) -> - H1 = if H == ?MAX_JSON_OBJ -> <<"<MAX>">>; + MAX_VAL = couch_views_encoding:max(), + H1 = if H == MAX_VAL -> <<"<MAX>">>; true -> H end, [H1 | maybe_replace_max_json(T)]; @@ -100,7 +99,7 @@ maybe_replace_max_json(EndKey) -> EndKey. -base_args(#cursor{index = Idx, selector = Selector} = Cursor) -> +base_args(#cursor{index = Idx} = Cursor) -> {StartKey, EndKey} = case Cursor#cursor.ranges of [empty] -> {null, null}; @@ -132,18 +131,19 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu #cursor{opts = Opts, bookmark = Bookmark} = Cursor, Args0 = apply_opts(Opts, BaseArgs), Args = mango_json_bookmark:update_args(Bookmark, Args0), - UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}), - DbOpts = [{user_ctx, UserCtx}], Result = case mango_idx:def(Idx) of all_docs -> CB = fun ?MODULE:handle_all_docs_message/2, - fabric:all_docs(Db, DbOpts, CB, Cursor, Args); + AllDocOpts = fabric2_util:all_docs_view_opts(Args) + ++ [{restart_tx, true}], + fabric2_db:fold_docs(Db, CB, Cursor, AllDocOpts); _ -> CB = fun ?MODULE:handle_message/2, % Normal view - DDoc = ddocid(Idx), + DDocId = mango_idx:ddoc(Idx), + {ok, DDoc} = fabric2_db:open_doc(Db, DDocId), Name = mango_idx:name(Idx), - fabric:query_view(Db, DbOpts, DDoc, Name, CB, Cursor, Args) + couch_views:query(Db, DDoc, Name, CB, Cursor, Args) end, case Result of {ok, LastCursor} -> @@ -227,7 +227,7 @@ choose_best_index(_DbName, IndexRanges) -> handle_message({meta, _}, Cursor) -> {ok, Cursor}; handle_message({row, Props}, Cursor) -> - case doc_member(Cursor, Props) of + case match_doc(Cursor, Props) of {ok, Doc, {execution_stats, Stats}} -> Cursor1 = Cursor#cursor { execution_stats = Stats @@ -280,15 +280,6 @@ handle_doc(C, _Doc) -> {stop, C}. -ddocid(Idx) -> - case mango_idx:ddoc(Idx) of - <<"_design/", Rest/binary>> -> - Rest; - Else -> - Else - end. - - apply_opts([], Args) -> Args; apply_opts([{conflicts, true} | Rest], Args) -> @@ -340,41 +331,18 @@ apply_opts([{_, _} | Rest], Args) -> apply_opts(Rest, Args). -doc_member(Cursor, RowProps) -> - Db = Cursor#cursor.db, - Opts = Cursor#cursor.opts, - ExecutionStats = Cursor#cursor.execution_stats, - Selector = Cursor#cursor.selector, - case couch_util:get_value(doc, RowProps) of - {DocProps} -> - % only matching documents are returned; the selector - % is evaluated at the shard level in view_cb({row, Row}, - {ok, {DocProps}, {execution_stats, ExecutionStats}}; - undefined -> - % an undefined doc was returned, indicating we should - % perform a quorum fetch - ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats), - couch_stats:increment_counter([mango, quorum_docs_examined]), - Id = couch_util:get_value(id, RowProps), - case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of - {ok, #doc{}=DocProps} -> - Doc = couch_doc:to_json_obj(DocProps, []), - match_doc(Selector, Doc, ExecutionStats1); - Else -> - Else - end; - _ -> - % no doc, no match - {no_match, null, {execution_stats, ExecutionStats}} - end. - - -match_doc(Selector, Doc, ExecutionStats) -> +match_doc(Cursor, RowProps) -> + #cursor{ + execution_stats = Stats0, + selector = Selector + } = Cursor, + Stats1 = mango_execution_stats:incr_docs_examined(Stats0, 1), + Doc = couch_util:get_value(doc, RowProps), case mango_selector:match(Selector, Doc) of true -> - {ok, Doc, {execution_stats, ExecutionStats}}; + {ok, Doc, {execution_stats, Stats1}}; false -> - {no_match, Doc, {execution_stats, ExecutionStats}} + {no_match, Doc, {execution_stats, Stats1}} end. diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl index 1054c74bb..94aa866d2 100644 --- a/src/mango/src/mango_httpd.erl +++ b/src/mango/src/mango_httpd.erl @@ -32,9 +32,8 @@ threshold = 1490 }). -handle_req(#httpd{} = Req, Db0) -> +handle_req(#httpd{} = Req, Db) -> try - Db = set_user_ctx(Req, Db0), handle_req_int(Req, Db) catch throw:{mango_error, Module, Reason} -> @@ -61,7 +60,9 @@ handle_req_int(_, _) -> handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) -> Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end, chttpd:qs(Req)), - Idxs = lists:sort(mango_idx:list(Db)), + Idxs = fabric2_fdb:transactional(Db, fun(TxDb) -> + lists:sort(mango_idx:list(TxDb)) + end), JsonIdxs0 = lists:map(fun mango_idx:to_json/1, Idxs), TotalRows = length(JsonIdxs0), Limit = case couch_util:get_value(limit, Params, TotalRows) of @@ -87,25 +88,27 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) -> {ok, Idx0} = mango_idx:new(Db, Opts), {ok, Idx} = mango_idx:validate_new(Idx0, Db), DbOpts = [{user_ctx, Req#httpd.user_ctx}, deleted, ejson_body], - {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), DbOpts), - Id = Idx#idx.ddoc, - Name = Idx#idx.name, - Status = case mango_idx:add(DDoc, Idx) of - {ok, DDoc} -> - <<"exists">>; - {ok, NewDDoc} -> - case mango_crud:insert(Db, NewDDoc, Opts) of - {ok, [{RespProps}]} -> - case lists:keyfind(error, 1, RespProps) of - {error, Reason} -> - ?MANGO_ERROR({error_saving_ddoc, Reason}); - _ -> - <<"created">> - end; - _ -> - ?MANGO_ERROR(error_saving_ddoc) - end - end, + Id = mango_idx:ddoc(Idx), + Name = mango_idx:name(Idx), + Status = fabric2_fdb:transactional(Db, fun(TxDb) -> + {ok, DDoc} = mango_util:load_ddoc(TxDb, Id, DbOpts), + case mango_idx:add(DDoc, Idx) of + {ok, DDoc} -> + <<"exists">>; + {ok, NewDDoc} -> + case mango_crud:insert(TxDb, NewDDoc, Opts) of + {ok, [{RespProps}]} -> + case lists:keyfind(error, 1, RespProps) of + {error, Reason} -> + ?MANGO_ERROR({error_saving_ddoc, Reason}); + _ -> + <<"created">> + end; + _ -> + ?MANGO_ERROR(error_saving_ddoc) + end + end + end), chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]}); handle_index_req(#httpd{path_parts=[_, _]}=Req, _Db) -> @@ -118,19 +121,21 @@ handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>, <<"_bulk_delete">>]}=Req, Db) -> chttpd:validate_ctype(Req, "application/json"), {ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)), - Idxs = mango_idx:list(Db), - DDocs = get_bulk_delete_ddocs(Opts), - {Success, Fail} = lists:foldl(fun(DDocId0, {Success0, Fail0}) -> - DDocId = convert_to_design_id(DDocId0), - Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end, - Id = {<<"id">>, DDocId}, - case mango_idx:delete(Filt, Db, Idxs, Opts) of - {ok, true} -> - {[{[Id, {<<"ok">>, true}]} | Success0], Fail0}; - {error, Error} -> - {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]} - end - end, {[], []}, DDocs), + {Success, Fail} = fabric2_fdb:transactional(Db, fun (TxDb) -> + Idxs = mango_idx:list(TxDb), + DDocs = get_bulk_delete_ddocs(Opts), + lists:foldl(fun(DDocId0, {Success0, Fail0}) -> + DDocId = convert_to_design_id(DDocId0), + Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end, + Id = {<<"id">>, DDocId}, + case mango_idx:delete(Filt, TxDb, Idxs, Opts) of + {ok, true} -> + {[{[Id, {<<"ok">>, true}]} | Success0], Fail0}; + {error, Error} -> + {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]} + end + end, {[], []}, DDocs) + end), chttpd:send_json(Req, {[{<<"success">>, Success}, {<<"fail">>, Fail}]}); handle_index_req(#httpd{path_parts=[_, <<"_index">>, @@ -144,15 +149,18 @@ handle_index_req(#httpd{method='DELETE', handle_index_req(#httpd{method='DELETE', path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) -> - Idxs = mango_idx:list(Db), - DDocId = convert_to_design_id(DDocId0), - Filt = fun(Idx) -> - IsDDoc = mango_idx:ddoc(Idx) == DDocId, - IsType = mango_idx:type(Idx) == Type, - IsName = mango_idx:name(Idx) == Name, - IsDDoc andalso IsType andalso IsName - end, - case mango_idx:delete(Filt, Db, Idxs, []) of + Result = fabric2_fdb:transactional(Db, fun(TxDb) -> + Idxs = mango_idx:list(TxDb), + DDocId = convert_to_design_id(DDocId0), + Filt = fun(Idx) -> + IsDDoc = mango_idx:ddoc(Idx) == DDocId, + IsType = mango_idx:type(Idx) == Type, + IsName = mango_idx:name(Idx) == Name, + IsDDoc andalso IsType andalso IsName + end, + mango_idx:delete(Filt, TxDb, Idxs, []) + end), + case Result of {ok, true} -> chttpd:send_json(Req, {[{ok, true}]}); {error, not_found} -> @@ -170,7 +178,9 @@ handle_explain_req(#httpd{method='POST'}=Req, Db) -> Body = chttpd:json_body_obj(Req), {ok, Opts0} = mango_opts:validate_find(Body), {value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0), - Resp = mango_crud:explain(Db, Sel, Opts), + Resp = fabric2_fdb:transactional(Db, fun(TxDb) -> + mango_crud:explain(TxDb, Sel, Opts) + end), chttpd:send_json(Req, Resp); handle_explain_req(Req, _Db) -> @@ -195,11 +205,6 @@ handle_find_req(Req, _Db) -> chttpd:send_method_not_allowed(Req, "POST"). -set_user_ctx(#httpd{user_ctx=Ctx}, Db) -> - {ok, NewDb} = couch_db:set_user_ctx(Db, Ctx), - NewDb. - - get_bulk_delete_ddocs(Opts) -> case lists:keyfind(docids, 1, Opts) of {docids, DDocs} when is_list(DDocs) -> diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl index ba9f68fd0..99a35886f 100644 --- a/src/mango/src/mango_idx.erl +++ b/src/mango/src/mango_idx.erl @@ -86,7 +86,7 @@ get_usable_indexes(Db, Selector, Opts) -> GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector( ExistingIndexes ), - BuiltIndexes = mango_cursor:remove_unbuilt_indexes(GlobalIndexes), + BuiltIndexes = remove_unbuilt_indexes(GlobalIndexes), UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts), UsableIndexes0 = lists:usort(BuiltIndexes ++ UserSpecifiedIndex), @@ -399,6 +399,17 @@ get_legacy_selector(Def) -> Selector -> Selector end. +% remove any interactive indexes that are not built. If an index is not +% interactive than we do not remove it as it will be built when queried +remove_unbuilt_indexes(Indexes) -> + lists:filter(fun(Idx) -> + case Idx#idx.interactive of + true -> Idx#idx.build_status == ?INDEX_READY; + _ -> true + end + end, Indexes). + + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl index 84be4180b..f80cc217b 100644 --- a/src/mango/src/mango_idx_view.erl +++ b/src/mango/src/mango_idx_view.erl @@ -34,7 +34,6 @@ -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). -include("mango_idx.hrl"). --include("mango_idx_view.hrl"). validate_new(#idx{}=Idx, _Db) -> @@ -183,11 +182,11 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) -> end_key([]) -> - [?MAX_JSON_OBJ]; + [couch_views_encoding:max()]; end_key([{_, _, '$lt', Key} | Rest]) -> case mango_json:special(Key) of true -> - [?MAX_JSON_OBJ]; + [couch_views_encoding:max()]; false -> [Key | end_key(Rest)] end; diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl deleted file mode 100644 index 0d213e56e..000000000 --- a/src/mango/src/mango_idx_view.hrl +++ /dev/null @@ -1,13 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
\ No newline at end of file diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl index 0d31f15f9..18a643985 100644 --- a/src/mango/src/mango_util.erl +++ b/src/mango/src/mango_util.erl @@ -85,14 +85,7 @@ open_doc(Db, DocId) -> open_doc(Db, DocId, Options) -> - case mango_util:defer(fabric, open_doc, [Db, DocId, Options]) of - {ok, Doc} -> - {ok, Doc}; - {not_found, _} -> - not_found; - _ -> - ?MANGO_ERROR({error_loading_doc, DocId}) - end. + fabric2_db:open_doc(Db, DocId, Options). open_ddocs(Db) -> @@ -111,7 +104,7 @@ load_ddoc(Db, DDocId, DbOpts) -> case open_doc(Db, DDocId, DbOpts) of {ok, Doc} -> {ok, check_lang(Doc)}; - not_found -> + {not_found, missing} -> Body = {[ {<<"language">>, <<"query">>} ]}, |