diff options
author | Nick Vatamaniuc <vatamane@gmail.com> | 2021-04-14 02:12:40 -0400 |
---|---|---|
committer | Nick Vatamaniuc <nickva@users.noreply.github.com> | 2021-04-16 17:44:43 -0400 |
commit | 5ec21191ad2956d393132e6fc3f11711e62d8d2c (patch) | |
tree | afa6602ecdf659f459e989d13ef285178a6ab92e | |
parent | 3080cf561c1d120a01e418a2acecfaa78dc182d0 (diff) | |
download | couchdb-5ec21191ad2956d393132e6fc3f11711e62d8d2c.tar.gz |
Move utilities and records from couch_mrview and couch_index to couch_views
* `couch_mrview_util` functions ended up mostly in `couch_views_util`.
* `couch_mrview` validatation functions ended up in `couch_views_validate`
module.
* `couch_mrview_http` functions moved to `couch_views_http_util`. The reason
they didn't end up in `couch_views_http` is because a lot of the functions
there have the exact same names as the ones in `couch_views_http_util`.
There is quite a bit of duplication involved but that is left for another
refactoring in the future. The general flow of control goes from chttpd ->
couch_views_http -> couch_views_http_util.
Most of the changes are just copy and paste with the exception of the
`ddoc_to_mrst/2` function. Previously, there were two almost identical copies
-- one in `couch_mrview_util` and another in `couch_views_util`. Both were used
by different parts of the code. The difference was the couch_views one
optionally disabled reduce functions, and replaced their body with the
`disabled` atom, while the one in `couch_mrview` didn't. Trying to unify them
such that only the `couch_views` one is used, resulted in the inability to
write design documents on server which have custom reduce disabled. That may be
a better behavior, however that should be updated in a separate PR and possibly
a mailing list discussion. So in order to preserve the exisiting behavior,
couch_eval was update to not fail in `try_compile` when design documents are
disabled.
Patches to the rest of the code to update the include path and use the new
utility functions will be updated in a separate commit.
-rw-r--r-- | src/couch_eval/src/couch_eval.erl | 6 | ||||
-rw-r--r-- | src/couch_views/include/couch_views.hrl | 94 | ||||
-rw-r--r-- | src/couch_views/src/couch_views_http_util.erl | 358 | ||||
-rw-r--r-- | src/couch_views/src/couch_views_util.erl | 105 | ||||
-rw-r--r-- | src/couch_views/src/couch_views_validate.erl | 460 |
5 files changed, 1018 insertions, 5 deletions
diff --git a/src/couch_eval/src/couch_eval.erl b/src/couch_eval/src/couch_eval.erl index a6e59657b..f87ba97b9 100644 --- a/src/couch_eval/src/couch_eval.erl +++ b/src/couch_eval/src/couch_eval.erl @@ -37,7 +37,7 @@ -type result() :: {doc_id(), [[{any(), any()}]]}. -type api_mod() :: atom(). -type context() :: {api_mod(), any()}. --type function_type() :: binary(). +-type function_type() :: binary() | atom(). -type function_name() :: binary(). -type function_src() :: binary(). -type error(_Error) :: no_return(). @@ -117,6 +117,10 @@ with_context(#{language := Language}, Fun) -> -spec try_compile(context(), function_type(), function_name(), function_src()) -> ok. +try_compile({_ApiMod, _Ctx}, reduce, <<_/binary>>, disabled) -> + % Reduce functions may be disabled. Accept that as a valid configuration. + ok; + try_compile({ApiMod, Ctx}, FuncType, FuncName, FuncSrc) -> ApiMod:try_compile(Ctx, FuncType, FuncName, FuncSrc). diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl index e28fa7478..86f73a325 100644 --- a/src/couch_views/include/couch_views.hrl +++ b/src/couch_views/include/couch_views.hrl @@ -45,3 +45,97 @@ % be used. Use `null` so it can can be round-tripped through json serialization % with couch_jobs. -define(VIEW_CURRENT_VSN, null). + + +-record(mrst, { + sig=nil, + fd=nil, + fd_monitor, + db_name, + idx_name, + language, + design_opts=[], + partitioned=false, + lib, + views, + id_btree=nil, + update_seq=0, + purge_seq=0, + first_build, + partial_resp_pid, + doc_acc, + doc_queue, + write_queue, + qserver=nil +}). + + +-record(mrview, { + id_num, + update_seq=0, + purge_seq=0, + map_names=[], + reduce_funs=[], + def, + btree=nil, + options=[] +}). + + +-define(MAX_VIEW_LIMIT, 16#10000000). + +-record(mrargs, { + view_type, + reduce, + + preflight_fun, + + start_key, + start_key_docid, + end_key, + end_key_docid, + keys, + + direction = fwd, + limit = ?MAX_VIEW_LIMIT, + skip = 0, + group_level = 0, + group = undefined, + stable = false, + update = true, + multi_get = false, + inclusive_end = true, + include_docs = false, + doc_options = [], + update_seq=false, + conflicts, + callback, + sorted = true, + extra = [], + page_size = undefined, + bookmark=nil +}). + +-record(vacc, { + db, + req, + resp, + prepend, + etag, + should_close = false, + buffer = [], + bufsize = 0, + threshold = 1490, + row_sent = false, + meta_sent = false, + paginated = false, + meta = #{} +}). + + +-record(view_row, { + key, + id, + value, + doc +}). diff --git a/src/couch_views/src/couch_views_http_util.erl b/src/couch_views/src/couch_views_http_util.erl new file mode 100644 index 000000000..7af07266a --- /dev/null +++ b/src/couch_views/src/couch_views_http_util.erl @@ -0,0 +1,358 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +% The reason this module and couch_views_http exist is because they have +% functions which are named the same but do slightly different things. The +% general pattern is chttpd code would call into couch_view_http and those +% function will in turn call into this module. + +-module(couch_views_http_util). + +-export([ + prepend_val/1, + parse_body_and_query/2, + parse_body_and_query/3, + parse_params/2, + parse_params/3, + parse_params/4, + view_cb/2, + row_to_obj/1, + row_to_obj/2, + row_to_json/1, + row_to_json/2 +]). + +-include_lib("couch/include/couch_db.hrl"). +-include_lib("couch_views/include/couch_views.hrl"). + +%% these clauses start (and possibly end) the response +view_cb({error, Reason}, #vacc{resp=undefined}=Acc) -> + {ok, Resp} = chttpd:send_error(Acc#vacc.req, Reason), + {ok, Acc#vacc{resp=Resp}}; + +view_cb(complete, #vacc{resp=undefined}=Acc) -> + % Nothing in view + {ok, Resp} = chttpd:send_json(Acc#vacc.req, 200, {[{rows, []}]}), + {ok, Acc#vacc{resp=Resp}}; + +view_cb(Msg, #vacc{resp=undefined}=Acc) -> + %% Start response + Headers = [], + {ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers), + view_cb(Msg, Acc#vacc{resp=Resp, should_close=true}); + +%% --------------------------------------------------- + +%% From here on down, the response has been started. + +view_cb({error, Reason}, #vacc{resp=Resp}=Acc) -> + {ok, Resp1} = chttpd:send_delayed_error(Resp, Reason), + {ok, Acc#vacc{resp=Resp1}}; + +view_cb(complete, #vacc{resp=Resp, buffer=Buf, threshold=Max}=Acc) -> + % Finish view output and possibly end the response + {ok, Resp1} = chttpd:close_delayed_json_object(Resp, Buf, "\r\n]}", Max), + case Acc#vacc.should_close of + true -> + {ok, Resp2} = chttpd:end_delayed_json_response(Resp1), + {ok, Acc#vacc{resp=Resp2}}; + _ -> + {ok, Acc#vacc{resp=Resp1, meta_sent=false, row_sent=false, + prepend=",\r\n", buffer=[], bufsize=0}} + end; + +view_cb({meta, Meta}, #vacc{meta_sent=false, row_sent=false}=Acc) -> + % Sending metadata as we've not sent it or any row yet + Parts = case couch_util:get_value(total, Meta) of + undefined -> []; + Total -> [io_lib:format("\"total_rows\":~p", [Total])] + end ++ case couch_util:get_value(offset, Meta) of + undefined -> []; + Offset -> [io_lib:format("\"offset\":~p", [Offset])] + end ++ case couch_util:get_value(update_seq, Meta) of + undefined -> []; + null -> + ["\"update_seq\":null"]; + UpdateSeq when is_integer(UpdateSeq) -> + [io_lib:format("\"update_seq\":~B", [UpdateSeq])]; + UpdateSeq when is_binary(UpdateSeq) -> + [io_lib:format("\"update_seq\":\"~s\"", [UpdateSeq])] + end ++ ["\"rows\":["], + Chunk = [prepend_val(Acc), "{", string:join(Parts, ","), "\r\n"], + {ok, AccOut} = maybe_flush_response(Acc, Chunk, iolist_size(Chunk)), + {ok, AccOut#vacc{prepend="", meta_sent=true}}; + +view_cb({meta, _Meta}, #vacc{}=Acc) -> + %% ignore metadata + {ok, Acc}; + +view_cb({row, Row}, #vacc{meta_sent=false}=Acc) -> + %% sorted=false and row arrived before meta + % Adding another row + Chunk = [prepend_val(Acc), "{\"rows\":[\r\n", row_to_json(Row)], + maybe_flush_response(Acc#vacc{meta_sent=true, row_sent=true}, Chunk, iolist_size(Chunk)); + +view_cb({row, Row}, #vacc{meta_sent=true}=Acc) -> + % Adding another row + Chunk = [prepend_val(Acc), row_to_json(Row)], + maybe_flush_response(Acc#vacc{row_sent=true}, Chunk, iolist_size(Chunk)). + + +maybe_flush_response(#vacc{bufsize=Size, threshold=Max} = Acc, Data, Len) + when Size > 0 andalso (Size + Len) > Max -> + #vacc{buffer = Buffer, resp = Resp} = Acc, + {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer), + {ok, Acc#vacc{prepend = ",\r\n", buffer = Data, bufsize = Len, resp = R1}}; +maybe_flush_response(Acc0, Data, Len) -> + #vacc{buffer = Buf, bufsize = Size} = Acc0, + Acc = Acc0#vacc{ + prepend = ",\r\n", + buffer = [Buf | Data], + bufsize = Size + Len + }, + {ok, Acc}. + +prepend_val(#vacc{prepend=Prepend}) -> + case Prepend of + undefined -> + ""; + _ -> + Prepend + end. + + +row_to_json(Row) -> + ?JSON_ENCODE(row_to_obj(Row)). + + +row_to_json(Kind, Row) -> + ?JSON_ENCODE(row_to_obj(Kind, Row)). + + +row_to_obj(Row) -> + Id = couch_util:get_value(id, Row), + row_to_obj(Id, Row). + + +row_to_obj(error, Row) -> + % Special case for _all_docs request with KEYS to + % match prior behavior. + Key = couch_util:get_value(key, Row), + Val = couch_util:get_value(value, Row), + Reason = couch_util:get_value(reason, Row), + ReasonProp = if Reason == undefined -> []; true -> + [{reason, Reason}] + end, + {[{key, Key}, {error, Val}] ++ ReasonProp}; +row_to_obj(Id0, Row) -> + Id = case Id0 of + undefined -> []; + Id0 -> [{id, Id0}] + end, + Key = couch_util:get_value(key, Row, null), + Val = couch_util:get_value(value, Row), + Doc = case couch_util:get_value(doc, Row) of + undefined -> []; + Doc0 -> [{doc, Doc0}] + end, + {Id ++ [{key, Key}, {value, Val}] ++ Doc}. + + +parse_params(#httpd{}=Req, Keys) -> + parse_params(chttpd:qs(Req), Keys); +parse_params(Props, Keys) -> + Args = #mrargs{}, + parse_params(Props, Keys, Args). + + +parse_params(Props, Keys, Args) -> + parse_params(Props, Keys, Args, []). + +parse_params(Props, Keys, #mrargs{}=Args0, Options) -> + IsDecoded = lists:member(decoded, Options), + Args1 = case lists:member(keep_group_level, Options) of + true -> + Args0; + _ -> + % group_level set to undefined to detect if explicitly set by user + Args0#mrargs{keys=Keys, group=undefined, group_level=undefined} + end, + lists:foldl(fun({K, V}, Acc) -> + parse_param(K, V, Acc, IsDecoded) + end, Args1, Props). + + +parse_body_and_query(#httpd{method='POST'} = Req, Keys) -> + Props = chttpd:json_body_obj(Req), + parse_body_and_query(Req, Props, Keys); + +parse_body_and_query(Req, Keys) -> + parse_params(chttpd:qs(Req), Keys, #mrargs{keys=Keys, group=undefined, + group_level=undefined}, [keep_group_level]). + +parse_body_and_query(Req, {Props}, Keys) -> + Args = #mrargs{keys=Keys, group=undefined, group_level=undefined}, + BodyArgs = parse_params(Props, Keys, Args, [decoded]), + parse_params(chttpd:qs(Req), Keys, BodyArgs, [keep_group_level]). + +parse_param(Key, Val, Args, IsDecoded) when is_binary(Key) -> + parse_param(binary_to_list(Key), Val, Args, IsDecoded); +parse_param(Key, Val, Args, IsDecoded) -> + case Key of + "" -> + Args; + "reduce" -> + Args#mrargs{reduce=parse_boolean(Val)}; + "key" when IsDecoded -> + Args#mrargs{start_key=Val, end_key=Val}; + "key" -> + JsonKey = ?JSON_DECODE(Val), + Args#mrargs{start_key=JsonKey, end_key=JsonKey}; + "keys" when IsDecoded -> + Args#mrargs{keys=Val}; + "keys" -> + Args#mrargs{keys=?JSON_DECODE(Val)}; + "startkey" when IsDecoded -> + Args#mrargs{start_key=Val}; + "start_key" when IsDecoded -> + Args#mrargs{start_key=Val}; + "startkey" -> + Args#mrargs{start_key=?JSON_DECODE(Val)}; + "start_key" -> + Args#mrargs{start_key=?JSON_DECODE(Val)}; + "startkey_docid" -> + Args#mrargs{start_key_docid=couch_util:to_binary(Val)}; + "start_key_doc_id" -> + Args#mrargs{start_key_docid=couch_util:to_binary(Val)}; + "endkey" when IsDecoded -> + Args#mrargs{end_key=Val}; + "end_key" when IsDecoded -> + Args#mrargs{end_key=Val}; + "endkey" -> + Args#mrargs{end_key=?JSON_DECODE(Val)}; + "end_key" -> + Args#mrargs{end_key=?JSON_DECODE(Val)}; + "endkey_docid" -> + Args#mrargs{end_key_docid=couch_util:to_binary(Val)}; + "end_key_doc_id" -> + Args#mrargs{end_key_docid=couch_util:to_binary(Val)}; + "limit" -> + Args#mrargs{limit=parse_pos_int(Val)}; + "page_size" -> + Args#mrargs{page_size=parse_pos_int(Val)}; + "stale" when Val == "ok" orelse Val == <<"ok">> -> + Args#mrargs{stable=true, update=false}; + "stale" when Val == "update_after" orelse Val == <<"update_after">> -> + Args#mrargs{stable=true, update=lazy}; + "stale" -> + throw({query_parse_error, <<"Invalid value for `stale`.">>}); + "stable" when Val == "true" orelse Val == <<"true">> orelse Val == true -> + Args#mrargs{stable=true}; + "stable" when Val == "false" orelse Val == <<"false">> orelse Val == false -> + Args#mrargs{stable=false}; + "stable" -> + throw({query_parse_error, <<"Invalid value for `stable`.">>}); + "update" when Val == "true" orelse Val == <<"true">> orelse Val == true -> + Args#mrargs{update=true}; + "update" when Val == "false" orelse Val == <<"false">> orelse Val == false -> + Args#mrargs{update=false}; + "update" when Val == "lazy" orelse Val == <<"lazy">> -> + Args#mrargs{update=lazy}; + "update" -> + throw({query_parse_error, <<"Invalid value for `update`.">>}); + "descending" -> + case parse_boolean(Val) of + true -> Args#mrargs{direction=rev}; + _ -> Args#mrargs{direction=fwd} + end; + "skip" -> + Args#mrargs{skip=parse_pos_int(Val)}; + "group" -> + Args#mrargs{group=parse_boolean(Val)}; + "group_level" -> + Args#mrargs{group_level=parse_pos_int(Val)}; + "inclusive_end" -> + Args#mrargs{inclusive_end=parse_boolean(Val)}; + "include_docs" -> + Args#mrargs{include_docs=parse_boolean(Val)}; + "attachments" -> + case parse_boolean(Val) of + true -> + Opts = Args#mrargs.doc_options, + Args#mrargs{doc_options=[attachments|Opts]}; + false -> + Args + end; + "att_encoding_info" -> + case parse_boolean(Val) of + true -> + Opts = Args#mrargs.doc_options, + Args#mrargs{doc_options=[att_encoding_info|Opts]}; + false -> + Args + end; + "update_seq" -> + Args#mrargs{update_seq=parse_boolean(Val)}; + "conflicts" -> + Args#mrargs{conflicts=parse_boolean(Val)}; + "callback" -> + Args#mrargs{callback=couch_util:to_binary(Val)}; + "sorted" -> + Args#mrargs{sorted=parse_boolean(Val)}; + "partition" -> + Partition = couch_util:to_binary(Val), + couch_partition:validate_partition(Partition), + couch_views_util:set_extra(Args, partition, Partition); + _ -> + BKey = couch_util:to_binary(Key), + BVal = couch_util:to_binary(Val), + Args#mrargs{extra=[{BKey, BVal} | Args#mrargs.extra]} + end. + + +parse_boolean(true) -> + true; +parse_boolean(false) -> + false; + +parse_boolean(Val) when is_binary(Val) -> + parse_boolean(?b2l(Val)); + +parse_boolean(Val) -> + case string:to_lower(Val) of + "true" -> true; + "false" -> false; + _ -> + Msg = io_lib:format("Invalid boolean parameter: ~p", [Val]), + throw({query_parse_error, ?l2b(Msg)}) + end. + +parse_int(Val) when is_integer(Val) -> + Val; +parse_int(Val) -> + case (catch list_to_integer(Val)) of + IntVal when is_integer(IntVal) -> + IntVal; + _ -> + Msg = io_lib:format("Invalid value for integer: ~p", [Val]), + throw({query_parse_error, ?l2b(Msg)}) + end. + +parse_pos_int(Val) -> + case parse_int(Val) of + IntVal when IntVal >= 0 -> + IntVal; + _ -> + Fmt = "Invalid value for positive integer: ~p", + Msg = io_lib:format(Fmt, [Val]), + throw({query_parse_error, ?l2b(Msg)}) + end. diff --git a/src/couch_views/src/couch_views_util.erl b/src/couch_views/src/couch_views_util.erl index 70400203c..287d4bad6 100644 --- a/src/couch_views/src/couch_views_util.erl +++ b/src/couch_views/src/couch_views_util.erl @@ -19,12 +19,16 @@ validate_args/1, validate_args/2, is_paginated/1, - active_tasks_info/5 + active_tasks_info/5, + set_view_type/3, + set_extra/3, + get_view_queries/1, + get_view_keys/1, + extract_view/4 ]). -include_lib("couch/include/couch_db.hrl"). --include_lib("couch_mrview/include/couch_mrview.hrl"). -include("couch_views.hrl"). @@ -80,10 +84,53 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) -> design_opts=DesignOpts, partitioned=Partitioned }, - SigInfo = {Views1, Language, DesignOpts, couch_index_util:sort_lib(Lib)}, + SigInfo = {Views1, Language, DesignOpts, sort_lib(Lib)}, {ok, IdxState#mrst{sig=couch_hash:md5_hash(term_to_binary(SigInfo))}}. +set_view_type(_Args, _ViewName, []) -> + throw({not_found, missing_named_view}); + +set_view_type(Args, ViewName, [View | Rest]) -> + RedNames = [N || {N, _} <- View#mrview.reduce_funs], + case lists:member(ViewName, RedNames) of + true -> + case Args#mrargs.reduce of + false -> Args#mrargs{view_type=map}; + _ -> Args#mrargs{view_type=red} + end; + false -> + case lists:member(ViewName, View#mrview.map_names) of + true -> Args#mrargs{view_type=map}; + false -> set_view_type(Args, ViewName, Rest) + end + end. + + +set_extra(#mrargs{} = Args, Key, Value) -> + Extra0 = Args#mrargs.extra, + Extra1 = lists:ukeysort(1, [{Key, Value} | Extra0]), + Args#mrargs{extra = Extra1}. + + +extract_view(_Lang, _Args, _ViewName, []) -> + throw({not_found, missing_named_view}); + +extract_view(Lang, #mrargs{view_type=map}=Args, Name, [View | Rest]) -> + Names = View#mrview.map_names ++ [N || {N, _} <- View#mrview.reduce_funs], + case lists:member(Name, Names) of + true -> {map, View, Args}; + _ -> extract_view(Lang, Args, Name, Rest) + end; + +extract_view(Lang, #mrargs{view_type=red}=Args, Name, [View | Rest]) -> + RedNames = [N || {N, _} <- View#mrview.reduce_funs], + case lists:member(Name, RedNames) of + true -> {red, {index_of(Name, RedNames), Lang, View}, Args}; + false -> extract_view(Lang, Args, Name, Rest) + end. + + collate_fun(View) -> #mrview{ options = Options @@ -122,7 +169,7 @@ validate_args(Args) -> validate_args(Args, []). -% This is mostly a copy of couch_mrview_util:validate_args/1 but it doesn't +% This is mostly a copy of couch_validate:validate_args/1 but it doesn't % update start / end keys and also throws a not_implemented error for reduce % validate_args(#mrargs{} = Args, Opts) -> @@ -366,3 +413,53 @@ convert_seq_to_stamp(Seq) -> VS = integer_to_list(Stamp) ++ "-" ++ integer_to_list(Batch) ++ "-" ++ integer_to_list(DocNumber), list_to_binary(VS). + + +get_view_queries({Props}) -> + case couch_util:get_value(<<"queries">>, Props) of + undefined -> + undefined; + Queries when is_list(Queries) -> + Queries; + _ -> + throw({bad_request, "`queries` member must be an array."}) + end. + + +get_view_keys({Props}) -> + case couch_util:get_value(<<"keys">>, Props) of + undefined -> + undefined; + Keys when is_list(Keys) -> + Keys; + _ -> + throw({bad_request, "`keys` member must be an array."}) + end. + + +sort_lib({Lib}) -> + sort_lib(Lib, []). + +sort_lib([], LAcc) -> + lists:keysort(1, LAcc); + +sort_lib([{LName, {LObj}}|Rest], LAcc) -> + LSorted = sort_lib(LObj, []), % descend into nested object + sort_lib(Rest, [{LName, LSorted}|LAcc]); + +sort_lib([{LName, LCode}|Rest], LAcc) -> + sort_lib(Rest, [{LName, LCode}|LAcc]). + + +index_of(Key, List) -> + index_of(Key, List, 1). + + +index_of(_, [], _) -> + throw({error, missing_named_view}); + +index_of(Key, [Key | _], Idx) -> + Idx; + +index_of(Key, [_ | Rest], Idx) -> + index_of(Key, Rest, Idx+1). diff --git a/src/couch_views/src/couch_views_validate.erl b/src/couch_views/src/couch_views_validate.erl new file mode 100644 index 000000000..558f65d1b --- /dev/null +++ b/src/couch_views/src/couch_views_validate.erl @@ -0,0 +1,460 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_views_validate). + + +-export([ + validate_args/1, + validate_args/3, + validate_ddoc/2 +]). + + +-define(LOWEST_KEY, null). +-define(HIGHEST_KEY, {<<255, 255, 255, 255>>}). + + +-include_lib("couch/include/couch_db.hrl"). +-include("couch_views.hrl"). + + +% There is another almost identical validate_args in couch_views_util. They +% should probably be merged at some point in the future. +% +validate_args(Args) -> + GroupLevel = determine_group_level(Args), + Reduce = Args#mrargs.reduce, + case Reduce == undefined orelse is_boolean(Reduce) of + true -> ok; + _ -> mrverror(<<"Invalid `reduce` value.">>) + end, + + case {Args#mrargs.view_type, Reduce} of + {map, true} -> mrverror(<<"Reduce is invalid for map-only views.">>); + _ -> ok + end, + + case {Args#mrargs.view_type, GroupLevel, Args#mrargs.keys} of + {red, exact, _} -> ok; + {red, _, KeyList} when is_list(KeyList) -> + Msg = <<"Multi-key fetches for reduce views must use `group=true`">>, + mrverror(Msg); + _ -> ok + end, + + case Args#mrargs.keys of + Keys when is_list(Keys) -> ok; + undefined -> ok; + _ -> mrverror(<<"`keys` must be an array of strings.">>) + end, + + case {Args#mrargs.keys, Args#mrargs.start_key, + Args#mrargs.end_key} of + {undefined, _, _} -> ok; + {[], _, _} -> ok; + {[_|_], undefined, undefined} -> ok; + _ -> mrverror(<<"`keys` is incompatible with `key`" + ", `start_key` and `end_key`">>) + end, + + case Args#mrargs.start_key_docid of + undefined -> ok; + SKDocId0 when is_binary(SKDocId0) -> ok; + _ -> mrverror(<<"`start_key_docid` must be a string.">>) + end, + + case Args#mrargs.end_key_docid of + undefined -> ok; + EKDocId0 when is_binary(EKDocId0) -> ok; + _ -> mrverror(<<"`end_key_docid` must be a string.">>) + end, + + case Args#mrargs.direction of + fwd -> ok; + rev -> ok; + _ -> mrverror(<<"Invalid direction.">>) + end, + + case {Args#mrargs.limit >= 0, Args#mrargs.limit == undefined} of + {true, _} -> ok; + {_, true} -> ok; + _ -> mrverror(<<"`limit` must be a positive integer.">>) + end, + + case Args#mrargs.skip < 0 of + true -> mrverror(<<"`skip` must be >= 0">>); + _ -> ok + end, + + case {Args#mrargs.view_type, GroupLevel} of + {red, exact} -> ok; + {_, 0} -> ok; + {red, Int} when is_integer(Int), Int >= 0 -> ok; + {red, _} -> mrverror(<<"`group_level` must be >= 0">>); + {map, _} -> mrverror(<<"Invalid use of grouping on a map view.">>) + end, + + case Args#mrargs.stable of + true -> ok; + false -> ok; + _ -> mrverror(<<"Invalid value for `stable`.">>) + end, + + case Args#mrargs.update of + true -> ok; + false -> ok; + lazy -> ok; + _ -> mrverror(<<"Invalid value for `update`.">>) + end, + + case is_boolean(Args#mrargs.inclusive_end) of + true -> ok; + _ -> mrverror(<<"Invalid value for `inclusive_end`.">>) + end, + + case {Args#mrargs.view_type, Args#mrargs.include_docs} of + {red, true} -> mrverror(<<"`include_docs` is invalid for reduce">>); + {_, ID} when is_boolean(ID) -> ok; + _ -> mrverror(<<"Invalid value for `include_docs`">>) + end, + + case {Args#mrargs.view_type, Args#mrargs.conflicts} of + {_, undefined} -> ok; + {map, V} when is_boolean(V) -> ok; + {red, undefined} -> ok; + {map, _} -> mrverror(<<"Invalid value for `conflicts`.">>); + {red, _} -> mrverror(<<"`conflicts` is invalid for reduce views.">>) + end, + + SKDocId = case {Args#mrargs.direction, Args#mrargs.start_key_docid} of + {fwd, undefined} -> <<>>; + {rev, undefined} -> <<255>>; + {_, SKDocId1} -> SKDocId1 + end, + + EKDocId = case {Args#mrargs.direction, Args#mrargs.end_key_docid} of + {fwd, undefined} -> <<255>>; + {rev, undefined} -> <<>>; + {_, EKDocId1} -> EKDocId1 + end, + + case is_boolean(Args#mrargs.sorted) of + true -> ok; + _ -> mrverror(<<"Invalid value for `sorted`.">>) + end, + + Args#mrargs{ + start_key_docid=SKDocId, + end_key_docid=EKDocId, + group_level=GroupLevel + }. + + +validate_args(Db, DDoc, Args0) -> + {ok, State} = couch_views_util:ddoc_to_mrst(fabric2_db:name(Db), DDoc), + Args1 = apply_limit(State#mrst.partitioned, Args0), + validate_args(State, Args1). + + +validate_ddoc(#{} = Db, DDoc) -> + DbName = fabric2_db:name(Db), + IsPartitioned = fabric2_db:is_partitioned(Db), + validate_ddoc(DbName, IsPartitioned, DDoc). + + +% Private functions + +validate_ddoc(DbName, _IsDbPartitioned, DDoc) -> + ok = validate_ddoc_fields(DDoc#doc.body), + GetName = fun + (#mrview{map_names = [Name | _]}) -> Name; + (#mrview{reduce_funs = [{Name, _} | _]}) -> Name; + (_) -> null + end, + ValidateView = fun(Ctx, #mrview{def=MapSrc, reduce_funs=Reds}=View) -> + couch_eval:try_compile(Ctx, map, GetName(View), MapSrc), + lists:foreach(fun + ({_RedName, <<"_sum", _/binary>>}) -> + ok; + ({_RedName, <<"_count", _/binary>>}) -> + ok; + ({_RedName, <<"_stats", _/binary>>}) -> + ok; + ({_RedName, <<"_approx_count_distinct", _/binary>>}) -> + ok; + ({_RedName, <<"_", _/binary>> = Bad}) -> + Msg = ["`", Bad, "` is not a supported reduce function."], + throw({invalid_design_doc, Msg}); + ({RedName, RedSrc}) -> + couch_eval:try_compile(Ctx, reduce, RedName, RedSrc) + end, Reds) + end, + {ok, #mrst{ + language = Lang, + views = Views + }} = couch_views_util:ddoc_to_mrst(DbName, DDoc), + + Views =/= [] andalso couch_eval:with_context(#{language => Lang}, fun (Ctx) -> + lists:foreach(fun(V) -> ValidateView(Ctx, V) end, Views) + end), + ok. + + +validate_args(#mrst{} = State, Args0) -> + Args = validate_args(Args0), + + ViewPartitioned = State#mrst.partitioned, + Partition = get_extra(Args, partition), + + case {ViewPartitioned, Partition} of + {true, undefined} -> + Msg1 = <<"`partition` parameter is mandatory " + "for queries to this view.">>, + mrverror(Msg1); + {true, _} -> + apply_partition(Args, Partition); + {false, undefined} -> + Args; + {false, Value} when is_binary(Value) -> + Msg2 = <<"`partition` parameter is not " + "supported in this design doc">>, + mrverror(Msg2) + end. + + +validate_ddoc_fields(DDoc) -> + MapFuncType = map_function_type(DDoc), + lists:foreach(fun(Path) -> + validate_ddoc_fields(DDoc, Path) + end, [ + [{<<"filters">>, object}, {any, [object, string]}], + [{<<"language">>, string}], + [{<<"lists">>, object}, {any, [object, string]}], + [{<<"options">>, object}], + [{<<"options">>, object}, {<<"include_design">>, boolean}], + [{<<"options">>, object}, {<<"local_seq">>, boolean}], + [{<<"options">>, object}, {<<"partitioned">>, boolean}], + [{<<"rewrites">>, [string, array]}], + [{<<"shows">>, object}, {any, [object, string]}], + [{<<"updates">>, object}, {any, [object, string]}], + [{<<"validate_doc_update">>, string}], + [{<<"views">>, object}, {<<"lib">>, object}], + [{<<"views">>, object}, {any, object}, {<<"map">>, MapFuncType}], + [{<<"views">>, object}, {any, object}, {<<"reduce">>, string}] + ]), + require_map_function_for_views(DDoc), + ok. + + +require_map_function_for_views({Props}) -> + case couch_util:get_value(<<"views">>, Props) of + undefined -> ok; + {Views} -> + lists:foreach(fun + ({<<"lib">>, _}) -> ok; + ({Key, {Value}}) -> + case couch_util:get_value(<<"map">>, Value) of + undefined -> throw({invalid_design_doc, + <<"View `", Key/binary, "` must contain map function">>}); + _ -> ok + end + end, Views), + ok + end. + + +validate_ddoc_fields(DDoc, Path) -> + case validate_ddoc_fields(DDoc, Path, []) of + ok -> ok; + {error, {FailedPath0, Type0}} -> + FailedPath = iolist_to_binary(join(FailedPath0, <<".">>)), + Type = format_type(Type0), + throw({invalid_design_doc, + <<"`", FailedPath/binary, "` field must have ", + Type/binary, " type">>}) + end. + +validate_ddoc_fields(undefined, _, _) -> + ok; + +validate_ddoc_fields(_, [], _) -> + ok; + +validate_ddoc_fields({KVS}=Props, [{any, Type} | Rest], Acc) -> + lists:foldl(fun + ({Key, _}, ok) -> + validate_ddoc_fields(Props, [{Key, Type} | Rest], Acc); + ({_, _}, {error, _}=Error) -> + Error + end, ok, KVS); + +validate_ddoc_fields({KVS}=Props, [{Key, Type} | Rest], Acc) -> + case validate_ddoc_field(Props, {Key, Type}) of + ok -> + validate_ddoc_fields(couch_util:get_value(Key, KVS), + Rest, + [Key | Acc]); + error -> + {error, {[Key | Acc], Type}}; + {error, Key1} -> + {error, {[Key1 | Acc], Type}} + end. + + +validate_ddoc_field(undefined, Type) when is_atom(Type) -> + ok; + +validate_ddoc_field(_, any) -> + ok; + +validate_ddoc_field(Value, Types) when is_list(Types) -> + lists:foldl(fun + (_, ok) -> ok; + (Type, _) -> validate_ddoc_field(Value, Type) + end, error, Types); +validate_ddoc_field(Value, string) when is_binary(Value) -> + ok; + +validate_ddoc_field(Value, array) when is_list(Value) -> + ok; + +validate_ddoc_field({Value}, object) when is_list(Value) -> + ok; + +validate_ddoc_field(Value, boolean) when is_boolean(Value) -> + ok; + +validate_ddoc_field({Props}, {any, Type}) -> + validate_ddoc_field1(Props, Type); + +validate_ddoc_field({Props}, {Key, Type}) -> + validate_ddoc_field(couch_util:get_value(Key, Props), Type); + +validate_ddoc_field(_, _) -> + error. + + +validate_ddoc_field1([], _) -> + ok; + +validate_ddoc_field1([{Key, Value} | Rest], Type) -> + case validate_ddoc_field(Value, Type) of + ok -> + validate_ddoc_field1(Rest, Type); + error -> + {error, Key} + end. + + +map_function_type({Props}) -> + case couch_util:get_value(<<"language">>, Props) of + <<"query">> -> object; + _ -> string + end. + + +format_type(Type) when is_atom(Type) -> + ?l2b(atom_to_list(Type)); + +format_type(Types) when is_list(Types) -> + iolist_to_binary(join(lists:map(fun atom_to_list/1, Types), <<" or ">>)). + + +join(L, Sep) -> + join(L, Sep, []). + + +join([H|[]], _, Acc) -> + [H | Acc]; + +join([H|T], Sep, Acc) -> + join(T, Sep, [Sep, H | Acc]). + + +determine_group_level(#mrargs{group=undefined, group_level=undefined}) -> + 0; + +determine_group_level(#mrargs{group=false, group_level=undefined}) -> + 0; + +determine_group_level(#mrargs{group=false, group_level=Level}) when Level > 0 -> + mrverror(<<"Can't specify group=false and group_level>0 at the same time">>); + +determine_group_level(#mrargs{group=true, group_level=undefined}) -> + exact; + +determine_group_level(#mrargs{group_level=GroupLevel}) -> + GroupLevel. + + +mrverror(Mesg) -> + throw({query_parse_error, Mesg}). + + +apply_partition(#mrargs{keys=[{p, _, _} | _]} = Args, _Partition) -> + Args; % already applied + +apply_partition(#mrargs{keys=Keys} = Args, Partition) when Keys /= undefined -> + Args#mrargs{keys=[{p, Partition, K} || K <- Keys]}; + +apply_partition(#mrargs{start_key={p, _, _}, end_key={p, _, _}} = Args, _Partition) -> + Args; % already applied. + +apply_partition(Args, Partition) -> + #mrargs{ + direction = Dir, + start_key = StartKey, + end_key = EndKey + } = Args, + + {DefSK, DefEK} = case Dir of + fwd -> {?LOWEST_KEY, ?HIGHEST_KEY}; + rev -> {?HIGHEST_KEY, ?LOWEST_KEY} + end, + + SK0 = if StartKey /= undefined -> StartKey; true -> DefSK end, + EK0 = if EndKey /= undefined -> EndKey; true -> DefEK end, + + Args#mrargs{ + start_key = {p, Partition, SK0}, + end_key = {p, Partition, EK0} + }. + + +get_extra(#mrargs{} = Args, Key) -> + couch_util:get_value(Key, Args#mrargs.extra). + + +apply_limit(ViewPartitioned, Args) -> + Options = Args#mrargs.extra, + IgnorePQLimit = lists:keyfind(ignore_partition_query_limit, 1, Options), + LimitType = case {ViewPartitioned, IgnorePQLimit} of + {true, false} -> "partition_query_limit"; + {true, _} -> "query_limit"; + {false, _} -> "query_limit" + end, + + MaxLimit = config:get_integer("query_server_config", + LimitType, ?MAX_VIEW_LIMIT), + + % Set the highest limit possible if a user has not + % specified a limit + Args1 = case Args#mrargs.limit == ?MAX_VIEW_LIMIT of + true -> Args#mrargs{limit = MaxLimit}; + false -> Args + end, + + if Args1#mrargs.limit =< MaxLimit -> Args1; true -> + Fmt = "Limit is too large, must not exceed ~p", + mrverror(io_lib:format(Fmt, [MaxLimit])) + end. |