diff options
author | Eric Avdey <eiri@eiri.ca> | 2017-05-01 10:27:31 -0300 |
---|---|---|
committer | Nick Vatamaniuc <nickva@users.noreply.github.com> | 2017-07-17 22:18:05 -0400 |
commit | 860f23cb57e0af84b668609676ba93c5edb769d5 (patch) | |
tree | e7b45dc069f5e39ad2e0a3bc8c8dbe0d6da47a0b | |
parent | cc21e141f71bd4bc314cf08692a3aa3a2698db43 (diff) | |
download | couchdb-860f23cb57e0af84b668609676ba93c5edb769d5.tar.gz |
Fix _local_docs end-point
This is a second attempt to fix _local_docs end-point. The previous one didn't
work on big enough btree_local, because local btree doesn't have reduction fun,
so reuse of couch_db_updater:btree_by_id_reduce/2 was crashing on a bad match
as soon as btree_local was getting kp_node. Also using full fold to calculate
total_rows value turned out to be resource expensive when a database have
significant number of local documents.
This fix avoids calculating of total_rows and offset instead always setting
them to null and also setting to null update_seq when requested, since it
doesn't have meaning in context of local documents.
A fabric module fabric_view_all_docs.erl was copied and modified as
fabric_view_local_docs.erl, because re-using it for processing of both types of
the documents was getting rather convoluted.
Jira: COUCHDB-3337
-rw-r--r-- | src/couch/src/couch_db.erl | 2 | ||||
-rw-r--r-- | src/couch_mrview/src/couch_mrview.erl | 34 | ||||
-rw-r--r-- | src/couch_mrview/src/couch_mrview_http.erl | 2 | ||||
-rw-r--r-- | src/couch_mrview/src/couch_mrview_util.erl | 13 | ||||
-rw-r--r-- | src/couch_mrview/test/couch_mrview_local_docs_tests.erl | 24 | ||||
-rw-r--r-- | src/fabric/src/fabric_view_all_docs.erl | 59 |
6 files changed, 86 insertions, 48 deletions
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index 51c81724d..7a1afa750 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -1288,7 +1288,7 @@ enum_docs(Db, <<"_local">>, InFun, InAcc, Options) -> FoldFun = pipe([fun skip_deleted/4], InFun), {ok, _LastReduce, OutAcc} = couch_btree:fold( Db#db.local_tree, FoldFun, InAcc, Options), - {ok, 0, OutAcc}; + {ok, null, OutAcc}; enum_docs(Db, NS, InFun, InAcc, Options0) -> FoldFun = pipe([ fun skip_deleted/4, diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl index ff43c6187..45dd83d66 100644 --- a/src/couch_mrview/src/couch_mrview.erl +++ b/src/couch_mrview/src/couch_mrview.erl @@ -403,8 +403,9 @@ cleanup(Db) -> all_docs_fold(Db, #mrargs{keys=undefined}=Args, Callback, UAcc) -> + ReduceFun = get_reduce_fun(Args), Total = get_total_rows(Db, Args), - UpdateSeq = couch_db:get_update_seq(Db), + UpdateSeq = get_update_seq(Db, Args), Acc = #mracc{ db=Db, total_rows=Total, @@ -412,7 +413,7 @@ all_docs_fold(Db, #mrargs{keys=undefined}=Args, Callback, UAcc) -> skip=Args#mrargs.skip, callback=Callback, user_acc=UAcc, - reduce_fun=fun couch_mrview_util:all_docs_reduce_to_count/1, + reduce_fun=ReduceFun, update_seq=UpdateSeq, args=Args }, @@ -420,8 +421,9 @@ all_docs_fold(Db, #mrargs{keys=undefined}=Args, Callback, UAcc) -> {ok, Offset, FinalAcc} = couch_db:enum_docs(Db, fun map_fold/3, Acc, Opts), finish_fold(FinalAcc, [{total, Total}, {offset, Offset}]); all_docs_fold(Db, #mrargs{direction=Dir, keys=Keys0}=Args, Callback, UAcc) -> + ReduceFun = get_reduce_fun(Args), Total = get_total_rows(Db, Args), - UpdateSeq = couch_db:get_update_seq(Db), + UpdateSeq = get_update_seq(Db, Args), Acc = #mracc{ db=Db, total_rows=Total, @@ -429,7 +431,7 @@ all_docs_fold(Db, #mrargs{direction=Dir, keys=Keys0}=Args, Callback, UAcc) -> skip=Args#mrargs.skip, callback=Callback, user_acc=UAcc, - reduce_fun=fun couch_mrview_util:all_docs_reduce_to_count/1, + reduce_fun=ReduceFun, update_seq=UpdateSeq, args=Args }, @@ -653,18 +655,34 @@ make_meta(Args, UpdateSeq, Base) -> end. -get_total_rows(#db{local_tree = LocalTree} = Db, #mrargs{extra = Extra}) -> +get_reduce_fun(#mrargs{extra = Extra}) -> case couch_util:get_value(namespace, Extra) of <<"_local">> -> - FoldFun = fun(_, _, Acc) -> {ok, Acc + 1} end, - {ok, _, Total} = couch_btree:foldl(LocalTree, FoldFun, 0), - Total; + fun(_) -> null end; + _ -> + fun couch_mrview_util:all_docs_reduce_to_count/1 + end. + + +get_total_rows(#db{} = Db, #mrargs{extra = Extra}) -> + case couch_util:get_value(namespace, Extra) of + <<"_local">> -> + null; _ -> {ok, Info} = couch_db:get_db_info(Db), couch_util:get_value(doc_count, Info) end. +get_update_seq(#db{} = Db, #mrargs{extra = Extra}) -> + case couch_util:get_value(namespace, Extra) of + <<"_local">> -> + null; + _ -> + couch_db:get_update_seq(Db) + end. + + default_cb(complete, Acc) -> {ok, lists:reverse(Acc)}; default_cb({final, Info}, []) -> diff --git a/src/couch_mrview/src/couch_mrview_http.erl b/src/couch_mrview/src/couch_mrview_http.erl index 7e3fd78e3..a94f48df9 100644 --- a/src/couch_mrview/src/couch_mrview_http.erl +++ b/src/couch_mrview/src/couch_mrview_http.erl @@ -380,6 +380,8 @@ view_cb({meta, Meta}, #vacc{meta_sent=false, row_sent=false}=Acc) -> Offset -> [io_lib:format("\"offset\":~p", [Offset])] end ++ case couch_util:get_value(update_seq, Meta) of undefined -> []; + null -> + ["\"update_seq\":null"]; UpdateSeq when is_integer(UpdateSeq) -> [io_lib:format("\"update_seq\":~B", [UpdateSeq])]; UpdateSeq when is_binary(UpdateSeq) -> diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl index 4bbb7cb92..5c95f2e46 100644 --- a/src/couch_mrview/src/couch_mrview_util.erl +++ b/src/couch_mrview/src/couch_mrview_util.erl @@ -345,8 +345,7 @@ get_row_count(#mrview{btree=Bt}) -> {ok, Count}. -all_docs_reduce_to_count(Reductions0) -> - Reductions = maybe_convert_reductions(Reductions0), +all_docs_reduce_to_count(Reductions) -> Reduce = fun couch_db_updater:btree_by_id_reduce/2, {Count, _, _} = couch_btree:final_reduce(Reduce, Reductions), Count. @@ -866,16 +865,6 @@ maybe_load_doc(Db, Id, Val, #mrargs{doc_options=Opts}) -> doc_row(couch_index_util:load_doc(Db, docid_rev(Id, Val), []), Opts). -maybe_convert_reductions({KVs0, UserReductions}) -> - KVs = lists:map(fun maybe_convert_kv/1, KVs0), - {KVs, UserReductions}. - -maybe_convert_kv({<<"_local/", _/binary>> = DocId, _}) -> - #full_doc_info{id = DocId}; -maybe_convert_kv(DocInfo) -> - DocInfo. - - doc_row(null, _Opts) -> [{doc, null}]; doc_row(Doc, Opts) -> diff --git a/src/couch_mrview/test/couch_mrview_local_docs_tests.erl b/src/couch_mrview/test/couch_mrview_local_docs_tests.erl index c16f53c62..f18f66e4e 100644 --- a/src/couch_mrview/test/couch_mrview_local_docs_tests.erl +++ b/src/couch_mrview/test/couch_mrview_local_docs_tests.erl @@ -43,7 +43,8 @@ all_docs_test_() -> fun should_query_with_range/1, fun should_query_with_range_rev/1, fun should_query_with_limit_and_skip/1, - fun should_query_with_include_docs/1 + fun should_query_with_include_docs/1, + fun should_query_with_update_seq/1 ] } } @@ -53,7 +54,7 @@ all_docs_test_() -> should_query(Db) -> Result = run_query(Db, []), Expect = {ok, [ - {meta, [{total, 10}, {offset, 0}]}, + {meta, [{total, null}, {offset, null}]}, mk_row(1), mk_row(10), mk_row(2), @@ -73,7 +74,7 @@ should_query_with_range(Db) -> {end_key, <<"_local/5">>} ]), Expect = {ok, [ - {meta, [{total, 10}, {offset, 3}]}, + {meta, [{total, null}, {offset, null}]}, mk_row(3), mk_row(4), mk_row(5) @@ -87,7 +88,7 @@ should_query_with_range_rev(Db) -> {inclusive_end, true} ]), Expect = {ok, [ - {meta, [{total, 10}, {offset, 4}]}, + {meta, [{total, null}, {offset, null}]}, mk_row(5), mk_row(4), mk_row(3) @@ -101,7 +102,7 @@ should_query_with_limit_and_skip(Db) -> {skip, 3} ]), Expect = {ok, [ - {meta, [{total, 10}, {offset, 5}]}, + {meta, [{total, null}, {offset, null}]}, mk_row(5), mk_row(6), mk_row(7) @@ -117,11 +118,22 @@ should_query_with_include_docs(Db) -> {row, Doc0} = mk_row(8), Doc = Doc0 ++ [{doc, {[{<<"val">>, 8}]}}], Expect = {ok, [ - {meta, [{total, 10}, {offset, 8}]}, + {meta, [{total, null}, {offset, null}]}, {row, Doc} ]}, ?_assertEqual(Expect, Result). +should_query_with_update_seq(Db) -> + Result = run_query(Db, [ + {start_key, <<"_local/2">>}, + {limit, 1}, + {update_seq, true} + ]), + Expect = {ok, [ + {meta, [{total, null}, {offset, null}, {update_seq, null}]}, + mk_row(2) + ]}, + ?_assertEqual(Expect, Result). mk_row(IntId) -> Id = list_to_binary(io_lib:format("_local/~b", [IntId])), diff --git a/src/fabric/src/fabric_view_all_docs.erl b/src/fabric/src/fabric_view_all_docs.erl index 2ba6f0dc5..de21dde08 100644 --- a/src/fabric/src/fabric_view_all_docs.erl +++ b/src/fabric/src/fabric_view_all_docs.erl @@ -58,9 +58,9 @@ go(DbName, Options, QueryArgs, Callback, Acc0) -> limit = Limit, conflicts = Conflicts, skip = Skip, - keys = Keys0 + keys = Keys0, + extra = Extra } = QueryArgs, - {_, Ref0} = spawn_monitor(fun() -> exit(fabric:get_doc_count(DbName)) end), DocOptions1 = case Conflicts of true -> [conflicts|DocOptions0]; _ -> DocOptions0 @@ -81,20 +81,31 @@ go(DbName, Options, QueryArgs, Callback, Acc0) -> true -> lists:sublist(Keys2, Limit); false -> Keys2 end, - Timeout = fabric_util:all_docs_timeout(), - receive {'DOWN', Ref0, _, _, Result} -> - case Result of - {ok, TotalRows} -> - {ok, Acc1} = Callback({meta, [{total, TotalRows}]}, Acc0), - {ok, Acc2} = doc_receive_loop( - Keys3, queue:new(), SpawnFun, MaxJobs, Callback, Acc1 - ), - Callback(complete, Acc2); - Error -> - Callback({error, Error}, Acc0) - end - after Timeout -> - Callback(timeout, Acc0) + Resp = case couch_util:get_value(namespace, Extra, <<"_all_docs">>) of + <<"_local">> -> + {ok, null}; + _ -> + Timeout = fabric_util:all_docs_timeout(), + {_, Ref0} = spawn_monitor(fun() -> + exit(fabric:get_doc_count(DbName)) + end), + receive {'DOWN', Ref0, _, _, Result} -> + Result + after Timeout -> + timeout + end + end, + case Resp of + {ok, TotalRows} -> + {ok, Acc1} = Callback({meta, [{total, TotalRows}]}, Acc0), + {ok, Acc2} = doc_receive_loop( + Keys3, queue:new(), SpawnFun, MaxJobs, Callback, Acc1 + ), + Callback(complete, Acc2); + timeout -> + Callback(timeout, Acc0); + Error -> + Callback({error, Error}, Acc0) end. go(DbName, _Options, Workers, QueryArgs, Callback, Acc0) -> @@ -142,10 +153,11 @@ handle_message({meta, Meta0}, {Worker, From}, State) -> 0 = fabric_dict:lookup_element(Worker, Counters0), rexi:stream_ack(From), Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), - Total = Total0 + Tot, - Offset = Offset0 + Off, - UpdateSeq = case UpdateSeq0 of - nil -> nil; + Total = if Tot == null -> null; true -> Total0 + Tot end, + Offset = if Off == null -> null; true -> Offset0 + Off end, + UpdateSeq = case {UpdateSeq0, Seq} of + {nil, _} -> nil; + {_, null} -> null; _ -> [{Worker, Seq} | UpdateSeq0] end, case fabric_dict:any(0, Counters1) of @@ -157,11 +169,16 @@ handle_message({meta, Meta0}, {Worker, From}, State) -> offset = Offset }}; false -> - FinalOffset = erlang:min(Total, Offset+State#collector.skip), + FinalOffset = case Offset of + null -> null; + _ -> erlang:min(Total, Offset+State#collector.skip) + end, Meta = [{total, Total}, {offset, FinalOffset}] ++ case UpdateSeq of nil -> []; + null -> + [{update_seq, null}]; _ -> [{update_seq, fabric_view_changes:pack_seqs(UpdateSeq)}] end, |