From 06544b2da795dcdcc19ac2ca4c473f1793fc8765 Mon Sep 17 00:00:00 2001 From: Nick Vatamaniuc Date: Fri, 30 Aug 2019 14:16:31 -0400 Subject: Implement _all_docs/queries Also add mrargs validation to match what master does and provide some helpful feedback to the users. --- src/chttpd/src/chttpd_db.erl | 193 +++++++++++++++++-------------- src/couch_views/src/couch_views_util.erl | 145 ++++++++++++++++++++++- test/elixir/test/basics_test.exs | 40 +++++++ 3 files changed, 290 insertions(+), 88 deletions(-) diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index 42cbb7df8..183758512 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -805,113 +805,132 @@ db_req(#httpd{path_parts=[_, DocId | FileNameParts]}=Req, Db) -> db_attachment_req(Req, Db, DocId, FileNameParts). multi_all_docs_view(Req, Db, OP, Queries) -> + UserCtx = Req#httpd.user_ctx, Args0 = couch_mrview_http:parse_params(Req, undefined), Args1 = Args0#mrargs{view_type=map}, ArgQueries = lists:map(fun({Query}) -> QueryArg1 = couch_mrview_http:parse_params(Query, undefined, Args1, [decoded]), - QueryArgs2 = fabric_util:validate_all_docs_args(Db, QueryArg1), + QueryArgs2 = couch_views_util:validate_args(QueryArg1), set_namespace(OP, QueryArgs2) end, Queries), - Options = [{user_ctx, Req#httpd.user_ctx}], - VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n"}, - FirstChunk = "{\"results\":[", - {ok, Resp0} = chttpd:start_delayed_json_response(VAcc0#vacc.req, - 200, [], FirstChunk), - VAcc1 = VAcc0#vacc{resp=Resp0}, - VAcc2 = lists:foldl(fun(Args, Acc0) -> - {ok, Acc1} = fabric2_db:fold_docs(Db, Options, - fun view_cb/2, Acc0, Args), - Acc1 - end, VAcc1, ArgQueries), - {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"), + Max = chttpd:chunked_response_buffer_size(), + First = "{\"results\":[", + {ok, Resp0} = chttpd:start_delayed_json_response(Req, 200, [], First), + VAcc0 = #vacc{ + db = Db, + req = Req, + resp = Resp0, + threshold = Max, + prepend = "\r\n" + }, + VAcc1 = lists:foldl(fun + (#mrargs{keys = undefined} = Args, Acc0) -> + send_all_docs(Db, Args, UserCtx, Acc0); + (#mrargs{keys = Keys} = Args, Acc0) when is_list(Keys) -> + send_all_docs_keys(Db, Args, UserCtx, Acc0) + end, VAcc0, ArgQueries), + {ok, Resp1} = chttpd:send_delayed_chunk(VAcc1#vacc.resp, "\r\n]}"), chttpd:end_delayed_json_response(Resp1). + all_docs_view(Req, Db, Keys, OP) -> + UserCtx = Req#httpd.user_ctx, Args0 = couch_mrview_http:parse_params(Req, Keys), - Args1 = set_namespace(OP, Args0), + Args1 = Args0#mrargs{view_type=map}, + Args2 = couch_views_util:validate_args(Args1), + Args3 = set_namespace(OP, Args2), Max = chttpd:chunked_response_buffer_size(), VAcc0 = #vacc{ db = Db, req = Req, threshold = Max }, - case Args1#mrargs.keys of + case Args3#mrargs.keys of undefined -> - Options = all_docs_view_opts(Args1, Req), - Acc = {iter, Db, Args1, VAcc0}, - {ok, {iter, _, _, Resp}} = - fabric2_db:fold_docs(Db, fun view_cb/2, Acc, Options), - {ok, Resp#vacc.resp}; - Keys0 when is_list(Keys0) -> - Keys1 = apply_args_to_keylist(Args1, Keys0), - %% namespace can be _set_ to `undefined`, so we - %% want simulate enum here - NS = case couch_util:get_value(namespace, Args1#mrargs.extra) of - <<"_all_docs">> -> <<"_all_docs">>; - <<"_design">> -> <<"_design">>; - <<"_local">> -> <<"_local">>; - _ -> <<"_all_docs">> - end, - TotalRows = fabric2_db:get_doc_count(Db, NS), - Meta = case Args1#mrargs.update_seq of - true -> - UpdateSeq = fabric2_db:get_update_seq(Db), - [{update_seq, UpdateSeq}]; - false -> - [] - end ++ [{total, TotalRows}, {offset, null}], - {ok, VAcc1} = view_cb({meta, Meta}, VAcc0), - DocOpts = case Args1#mrargs.conflicts of - true -> [conflicts | Args1#mrargs.doc_options]; - _ -> Args1#mrargs.doc_options - end ++ [{user_ctx, Req#httpd.user_ctx}], - IncludeDocs = Args1#mrargs.include_docs, - VAcc2 = lists:foldl(fun(DocId, Acc) -> - OpenOpts = [deleted | DocOpts], - Row0 = case fabric2_db:open_doc(Db, DocId, OpenOpts) of - {not_found, missing} -> - #view_row{key = DocId}; - {ok, #doc{deleted = true, revs = Revs}} -> - {RevPos, [RevId | _]} = Revs, - Value = {[ - {rev, couch_doc:rev_to_str({RevPos, RevId})}, - {deleted, true} - ]}, - DocValue = if not IncludeDocs -> undefined; true -> - null - end, - #view_row{ - key = DocId, - id = DocId, - value = Value, - doc = DocValue - }; - {ok, #doc{revs = Revs} = Doc0} -> - {RevPos, [RevId | _]} = Revs, - Value = {[ - {rev, couch_doc:rev_to_str({RevPos, RevId})} - ]}, - DocValue = if not IncludeDocs -> undefined; true -> - couch_doc:to_json_obj(Doc0, DocOpts) - end, - #view_row{ - key = DocId, - id = DocId, - value = Value, - doc = DocValue - } - end, - Row1 = fabric_view:transform_row(Row0), - {ok, NewAcc} = view_cb(Row1, Acc), - NewAcc - end, VAcc1, Keys1), - {ok, VAcc3} = view_cb(complete, VAcc2), - {ok, VAcc3#vacc.resp} + VAcc1 = send_all_docs(Db, Args3, UserCtx, VAcc0), + {ok, VAcc1#vacc.resp}; + Keys when is_list(Keys) -> + VAcc1 = send_all_docs_keys(Db, Args3, UserCtx, VAcc0), + {ok, VAcc2} = view_cb(complete, VAcc1), + {ok, VAcc2#vacc.resp} end. -all_docs_view_opts(Args, Req) -> +send_all_docs(Db, #mrargs{keys = undefined} = Args, UserCtx, VAcc0) -> + Opts = all_docs_view_opts(Args, UserCtx), + Acc = {iter, Db, Args, VAcc0}, + ViewCb = fun view_cb/2, + {ok, {iter, _, _, VAcc1}} = fabric2_db:fold_docs(Db, ViewCb, Acc, Opts), + VAcc1. + + +send_all_docs_keys(Db, #mrargs{} = Args, UserCtx, VAcc0) -> + Keys = apply_args_to_keylist(Args, Args#mrargs.keys), + %% namespace can be _set_ to `undefined`, so we + %% want simulate enum here + NS = case couch_util:get_value(namespace, Args#mrargs.extra) of + <<"_all_docs">> -> <<"_all_docs">>; + <<"_design">> -> <<"_design">>; + <<"_local">> -> <<"_local">>; + _ -> <<"_all_docs">> + end, + TotalRows = fabric2_db:get_doc_count(Db, NS), + Meta = case Args#mrargs.update_seq of + true -> + UpdateSeq = fabric2_db:get_update_seq(Db), + [{update_seq, UpdateSeq}]; + false -> + [] + end ++ [{total, TotalRows}, {offset, null}], + {ok, VAcc1} = view_cb({meta, Meta}, VAcc0), + DocOpts = case Args#mrargs.conflicts of + true -> [conflicts | Args#mrargs.doc_options]; + _ -> Args#mrargs.doc_options + end ++ [{user_ctx, UserCtx}], + IncludeDocs = Args#mrargs.include_docs, + VAcc2 = lists:foldl(fun(DocId, Acc) -> + OpenOpts = [deleted | DocOpts], + Row0 = case fabric2_db:open_doc(Db, DocId, OpenOpts) of + {not_found, missing} -> + #view_row{key = DocId}; + {ok, #doc{deleted = true, revs = Revs}} -> + {RevPos, [RevId | _]} = Revs, + Value = {[ + {rev, couch_doc:rev_to_str({RevPos, RevId})}, + {deleted, true} + ]}, + DocValue = if not IncludeDocs -> undefined; true -> + null + end, + #view_row{ + key = DocId, + id = DocId, + value = Value, + doc = DocValue + }; + {ok, #doc{revs = Revs} = Doc0} -> + {RevPos, [RevId | _]} = Revs, + Value = {[ + {rev, couch_doc:rev_to_str({RevPos, RevId})} + ]}, + DocValue = if not IncludeDocs -> undefined; true -> + couch_doc:to_json_obj(Doc0, DocOpts) + end, + #view_row{ + key = DocId, + id = DocId, + value = Value, + doc = DocValue + } + end, + Row1 = fabric_view:transform_row(Row0), + {ok, NewAcc} = view_cb(Row1, Acc), + NewAcc + end, VAcc1, Keys). + + +all_docs_view_opts(Args, UserCtx) -> StartKey = case Args#mrargs.start_key of undefined -> Args#mrargs.start_key_docid; SKey -> SKey @@ -925,7 +944,7 @@ all_docs_view_opts(Args, Req) -> {_, _} -> [{end_key, EndKey}] end, [ - {user_ctx, Req#httpd.user_ctx}, + {user_ctx, UserCtx}, {dir, Args#mrargs.direction}, {start_key, StartKey}, {limit, Args#mrargs.limit}, diff --git a/src/couch_views/src/couch_views_util.erl b/src/couch_views/src/couch_views_util.erl index b88cfcd22..b9dbb71d1 100644 --- a/src/couch_views/src/couch_views_util.erl +++ b/src/couch_views/src/couch_views_util.erl @@ -14,7 +14,8 @@ -export([ - ddoc_to_mrst/2 + ddoc_to_mrst/2, + validate_args/1 ]). @@ -82,3 +83,145 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) -> }, SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)}, {ok, IdxState#mrst{sig=couch_hash:md5_hash(term_to_binary(SigInfo))}}. + + +% This is mostly a copy of couch_mrview_util:validate_args/1 but it doesn't +% update start / end keys and also throws a not_implemented error for reduce +% +validate_args(#mrargs{} = Args) -> + GroupLevel = determine_group_level(Args), + Reduce = Args#mrargs.reduce, + + case Reduce =/= undefined orelse Args#mrargs.view_type == red of + true -> throw(not_implemented); + false -> ok + end, + + case Reduce == undefined orelse is_boolean(Reduce) of + true -> ok; + _ -> mrverror(<<"Invalid `reduce` value.">>) + end, + + case {Args#mrargs.view_type, Reduce} of + {map, true} -> mrverror(<<"Reduce is invalid for map-only views.">>); + _ -> ok + end, + + case {Args#mrargs.view_type, GroupLevel, Args#mrargs.keys} of + {red, exact, _} -> ok; + {red, _, KeyList} when is_list(KeyList) -> + Msg = <<"Multi-key fetchs for reduce views must use `group=true`">>, + mrverror(Msg); + _ -> ok + end, + + case Args#mrargs.keys of + Keys when is_list(Keys) -> ok; + undefined -> ok; + _ -> mrverror(<<"`keys` must be an array of strings.">>) + end, + + case {Args#mrargs.keys, Args#mrargs.start_key, + Args#mrargs.end_key} of + {undefined, _, _} -> ok; + {[], _, _} -> ok; + {[_|_], undefined, undefined} -> ok; + _ -> mrverror(<<"`keys` is incompatible with `key`" + ", `start_key` and `end_key`">>) + end, + + case Args#mrargs.start_key_docid of + undefined -> ok; + SKDocId0 when is_binary(SKDocId0) -> ok; + _ -> mrverror(<<"`start_key_docid` must be a string.">>) + end, + + case Args#mrargs.end_key_docid of + undefined -> ok; + EKDocId0 when is_binary(EKDocId0) -> ok; + _ -> mrverror(<<"`end_key_docid` must be a string.">>) + end, + + case Args#mrargs.direction of + fwd -> ok; + rev -> ok; + _ -> mrverror(<<"Invalid direction.">>) + end, + + case {Args#mrargs.limit >= 0, Args#mrargs.limit == undefined} of + {true, _} -> ok; + {_, true} -> ok; + _ -> mrverror(<<"`limit` must be a positive integer.">>) + end, + + case Args#mrargs.skip < 0 of + true -> mrverror(<<"`skip` must be >= 0">>); + _ -> ok + end, + + case {Args#mrargs.view_type, GroupLevel} of + {red, exact} -> ok; + {_, 0} -> ok; + {red, Int} when is_integer(Int), Int >= 0 -> ok; + {red, _} -> mrverror(<<"`group_level` must be >= 0">>); + {map, _} -> mrverror(<<"Invalid use of grouping on a map view.">>) + end, + + case Args#mrargs.stable of + true -> ok; + false -> ok; + _ -> mrverror(<<"Invalid value for `stable`.">>) + end, + + case Args#mrargs.update of + true -> ok; + false -> ok; + lazy -> ok; + _ -> mrverror(<<"Invalid value for `update`.">>) + end, + + case is_boolean(Args#mrargs.inclusive_end) of + true -> ok; + _ -> mrverror(<<"Invalid value for `inclusive_end`.">>) + end, + + case {Args#mrargs.view_type, Args#mrargs.include_docs} of + {red, true} -> mrverror(<<"`include_docs` is invalid for reduce">>); + {_, ID} when is_boolean(ID) -> ok; + _ -> mrverror(<<"Invalid value for `include_docs`">>) + end, + + case {Args#mrargs.view_type, Args#mrargs.conflicts} of + {_, undefined} -> ok; + {map, V} when is_boolean(V) -> ok; + {red, undefined} -> ok; + {map, _} -> mrverror(<<"Invalid value for `conflicts`.">>); + {red, _} -> mrverror(<<"`conflicts` is invalid for reduce views.">>) + end, + + case is_boolean(Args#mrargs.sorted) of + true -> ok; + _ -> mrverror(<<"Invalid value for `sorted`.">>) + end, + + Args#mrargs{group_level=GroupLevel}. + + +determine_group_level(#mrargs{group=undefined, group_level=undefined}) -> + 0; + +determine_group_level(#mrargs{group=false, group_level=undefined}) -> + 0; + +determine_group_level(#mrargs{group=false, group_level=Level}) when Level > 0 -> + mrverror(<<"Can't specify group=false and group_level>0 at the same time">>); + +determine_group_level(#mrargs{group=true, group_level=undefined}) -> + exact; + +determine_group_level(#mrargs{group_level=GroupLevel}) -> + GroupLevel. + + +mrverror(Mesg) -> + throw({query_parse_error, Mesg}). diff --git a/test/elixir/test/basics_test.exs b/test/elixir/test/basics_test.exs index 363972b2a..70dd6e84c 100644 --- a/test/elixir/test/basics_test.exs +++ b/test/elixir/test/basics_test.exs @@ -316,4 +316,44 @@ defmodule BasicsTest do # TODO assert true end + + @tag :with_db + test "_all_docs/queries works", context do + db_name = context[:db_name] + + resp = Couch.post("/#{db_name}/_all_docs/queries", body: %{:queries => []}) + assert resp.status_code == 200 + assert resp.body["results"] == [] + + assert Couch.put("/#{db_name}/doc1", body: %{:a => 1}).body["ok"] + + body = %{ + :queries => [ + %{:limit => 1}, + %{:limit => 0} + ] + } + resp = Couch.post("/#{db_name}/_all_docs/queries", body: body) + assert resp.status_code == 200 + + assert Map.has_key?(resp.body, "results") + results = Enum.sort(resp.body["results"]) + assert length(results) == 2 + [res1, res2] = results + + assert res1 == %{"offset" => :null, "rows" => [], "total_rows" => 1} + + assert res2["offset"] == :null + assert res2["total_rows"] == 1 + rows = res2["rows"] + + assert length(rows) == 1 + [row] = rows + assert row["id"] == "doc1" + assert row["key"] == "doc1" + + val = row["value"] + assert Map.has_key?(val, "rev") + end + end -- cgit v1.2.1